From 9d46be270a78d7856c900f47fd7832937019970f Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 31 May 2026 16:41:56 -0400 Subject: [PATCH 01/13] Start implementing blocked leaf pages. --- conan.lock | 10 +- conanfile.py | 1 + src/turtle_kv/core/edit_view.hpp | 8 + src/turtle_kv/core/key_view.hpp | 9 + src/turtle_kv/core/packed_key_value_slot.hpp | 184 ++++++++++++++++++ src/turtle_kv/core/value_view.hpp | 10 + src/turtle_kv/import/buffer.hpp | 5 +- src/turtle_kv/mem_table/mem_table.test.cpp | 6 +- src/turtle_kv/tree/packed_leaf_block.hpp | 175 +++++++++++++++++ src/turtle_kv/tree/packed_leaf_block.ipp | 100 ++++++++++ src/turtle_kv/tree/packed_leaf_block.test.cpp | 121 ++++++++++++ src/turtle_kv/tree/packed_leaf_page2.hpp | 37 ++++ src/turtle_kv/tree/packed_node_page.cpp | 6 +- 13 files changed, 658 insertions(+), 14 deletions(-) create mode 100644 src/turtle_kv/tree/packed_leaf_block.hpp create mode 100644 src/turtle_kv/tree/packed_leaf_block.ipp create mode 100644 src/turtle_kv/tree/packed_leaf_block.test.cpp create mode 100644 src/turtle_kv/tree/packed_leaf_page2.hpp diff --git a/conan.lock b/conan.lock index 13b562b..1607f64 100644 --- a/conan.lock +++ b/conan.lock @@ -2,7 +2,8 @@ "version": "0.5", "requires": [ "abseil/20250127.0", - "batteries/0.70.2", + "artc/0.0.1.dev", + "batteries/0.70.4.dev2", "boost/1.88.0", "bzip2/1.0.8", "cli11/2.5.0", @@ -19,7 +20,7 @@ "openssl/3.6.0", "pcg-cpp/cci.20220409", "protobuf/3.21.12", - "vqf/0.2.5", + "vqf/0.2.6", "xxhash/0.8.3", "yaml-cpp/0.9.0", "zlib/1.3.1" @@ -32,8 +33,7 @@ "ninja/1.13.2" ], "python_requires": [ - "cor_recipe_utils/0.18.2", - "cor_recipe_utils/0.8.7" + "cor_recipe_utils/0.19.1" ], "overrides": { "libunwind/[>=1.8 <2]": [ @@ -52,7 +52,7 @@ "boost/1.88.0" ], "batteries/[>=0.60.2 <2]": [ - "batteries/0.70.2" + "batteries/0.70.4.dev2" ] }, "config_requires": [] diff --git a/conanfile.py b/conanfile.py index 31cc052..6892cf6 100644 --- a/conanfile.py +++ b/conanfile.py @@ -85,6 +85,7 @@ def requirements(self): } self.requires("abseil/20250127.0", **VISIBLE, **OVERRIDE) + self.requires("artc/[>=0.0.1 <1]") self.requires("batteries/[>=0.70.2 <1]", **VISIBLE, **OVERRIDE) self.requires("boost/1.88.0", **VISIBLE, **OVERRIDE) self.requires("glog/0.7.1", **VISIBLE) diff --git a/src/turtle_kv/core/edit_view.hpp b/src/turtle_kv/core/edit_view.hpp index b5e6f62..9fc9c20 100644 --- a/src/turtle_kv/core/edit_view.hpp +++ b/src/turtle_kv/core/edit_view.hpp @@ -1,3 +1,11 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #pragma once #include diff --git a/src/turtle_kv/core/key_view.hpp b/src/turtle_kv/core/key_view.hpp index 7c91fa8..bbf16b6 100644 --- a/src/turtle_kv/core/key_view.hpp +++ b/src/turtle_kv/core/key_view.hpp @@ -85,4 +85,13 @@ inline usize packed_key_data_size(const KeyView& key) return key.size(); } +template +concept HasKeyView = requires(const T& obj) { + { get_key(obj) } -> std::convertible_to; +}; + +static_assert(HasKeyView); +static_assert(HasKeyView); +static_assert(HasKeyView); + } // namespace turtle_kv diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index 9bc08d2..0f0d2d2 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -17,8 +17,146 @@ #include #include +#include + namespace turtle_kv { +struct PackedKeyValueSlot; + +using PackedKeyValueSlotPtr = llfs::PackedPointer; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedKeyValueSlot { + little_u16 key_size; + char key_data_[0]; + + //----- --- -- - - - - + // u8 key_bytes[this->key_size] + //----- --- -- - - - - + // u8 op_code + // u8 value_bytes[this->item_size - offsetof(this->value_bytes)] + //----- --- -- - - - - + + //+++++++++++-+-+--+----- --- -- - - - - + + PackedKeyValueSlot(const PackedKeyValueSlot&) = delete; + PackedKeyValueSlot& operator=(const PackedKeyValueSlot&) = delete; + + //+++++++++++-+-+--+----- --- -- - - - - + + usize slot_size(const PackedKeyValueSlotPtr* p_this) const noexcept + { + const PackedKeyValueSlotPtr* const p_next = p_this + 1; + return usize{p_next->offset.value()} - usize{p_this->offset.value()} + + sizeof(PackedKeyValueSlotPtr); + } + + const char* key_data() const noexcept + { + return this->key_data_; + } + + KeyView key_view() const noexcept + { + return KeyView{this->key_data_, this->key_size}; + } + + const char* value_data() const noexcept + { + return this->key_data() + (this->key_size + 1); + } + + const char* value_data_end(const PackedKeyValueSlotPtr* p_this) const noexcept + { + return this->value_data_end(/*size_of_slot=*/this->slot_size(p_this)); + } + + const char* value_data_end(usize size_of_slot) const noexcept + { + return reinterpret_cast(this) + size_of_slot; + } + + usize value_size(const PackedKeyValueSlotPtr* p_this) const noexcept + { + return this->value_size(/*size_of_slot=*/this->slot_size(p_this)); + } + + usize value_size(usize size_of_slot) const noexcept + { + return this->value_data_end(size_of_slot) - this->value_data(); + } + + ValueView::OpCode value_op_code() const noexcept + { + return static_cast(this->key_data_[this->key_size]); + } + + ValueView value_view(const PackedKeyValueSlotPtr* p_this) const noexcept + { + return this->value_view(/*size_of_slot=*/this->slot_size(p_this)); + } + + ValueView value_view(usize size_of_slot) const noexcept + { + return ValueView::from_packed( + this->value_op_code(), + std::string_view{this->value_data(), this->value_size(size_of_slot)}); + } +}; + +inline KeyView get_key(const PackedKeyValueSlot& packed_slot) noexcept +{ + return packed_slot.key_view(); +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// TODO [tastolfi 2026-05-31] use 16-bit pointer tagging to store slot_size with the pointer in one +// 64-bit word. +// +struct PackedKeyValueSlotRef { + const PackedKeyValueSlot* p_slot; + usize slot_size; +}; + +inline KeyView get_key(const PackedKeyValueSlotRef& slot_ref) noexcept +{ + return slot_ref.p_slot->key_view(); +} + +inline ValueView get_value(const PackedKeyValueSlotRef& slot_ref) noexcept +{ + return slot_ref.p_slot->value_view(slot_ref.slot_size); +} + +inline const PackedKeyValueSlotRef& to_key_value_slot_ref(const PackedKeyValueSlotRef& ref) noexcept +{ + return ref; +} + +inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr* pp_slot) noexcept +{ + return PackedKeyValueSlotRef{ + .p_slot = pp_slot->get(), + .slot_size = pp_slot->get()->slot_size(pp_slot), + }; +} + +inline PackedKeyValueSlotRef to_key_value_slot_ref(const ConstBuffer& slot_buffer) noexcept +{ + return PackedKeyValueSlotRef{ + .p_slot = static_cast(slot_buffer.data()), + .slot_size = slot_buffer.size(), + }; +} + +template +concept ConvertibleToKeyValueSlotRef = requires(const T& obj) { + { to_key_value_slot_ref(obj) } -> std::convertible_to; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// /** \brief Returns the size required (in bytes) to pack a slot with the passed key and * value. */ @@ -30,6 +168,21 @@ inline usize packed_key_value_slot_size(const KeyView& key, const ValueView& val + value.size(); } +template +inline usize packed_key_value_slot_size(const T& obj) noexcept +{ + return to_key_value_slot_ref(obj).slot_size; +} + +template + requires HasKeyView && HasValueView +inline usize packed_key_value_slot_size(const T& obj) noexcept +{ + return packed_key_value_slot_size(get_key(obj), get_value(obj)); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// /** \brief Serializes the passed key and value into the destination buffer. */ inline std::pair pack_key_value_slot(const KeyView& key, @@ -72,6 +225,37 @@ inline std::pair pack_key_value_slot(const KeyView& key, })); } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief + */ +template + requires HasKeyView && HasValueView +inline usize pack_key_value_slot(const T& src, void* dst) noexcept +{ + const KeyView& key = get_key(src); + const ValueView& value = get_value(src); + const usize slot_size = packed_key_value_slot_size(key, value); + + pack_key_value_slot(key, value, MutableBuffer{dst, slot_size}); + + return slot_size; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief + */ +template +inline usize pack_key_value_slot(const T& src, void* dst) noexcept +{ + const PackedKeyValueSlotRef& slot_ref = to_key_value_slot_ref(src); + std::memcpy(dst, slot_ref.p_slot, slot_ref.slot_size); + return slot_ref.slot_size; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// /** \brief Unpacks a key/value pair from the passed packed slot buffer. */ inline StatusOr> unpack_key_value_slot(ConstBuffer payload) diff --git a/src/turtle_kv/core/value_view.hpp b/src/turtle_kv/core/value_view.hpp index e79673f..3fb4d7b 100644 --- a/src/turtle_kv/core/value_view.hpp +++ b/src/turtle_kv/core/value_view.hpp @@ -404,4 +404,14 @@ inline bool decays_to_item(const ValueView& value) return false; } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// + +template +concept HasValueView = requires(const T& obj) { + { get_value(obj) } -> std::convertible_to; +}; + +static_assert(HasValueView); + } // namespace turtle_kv diff --git a/src/turtle_kv/import/buffer.hpp b/src/turtle_kv/import/buffer.hpp index 8640034..0d712e7 100644 --- a/src/turtle_kv/import/buffer.hpp +++ b/src/turtle_kv/import/buffer.hpp @@ -6,14 +6,13 @@ namespace turtle_kv { +using batt::advance_pointer; using batt::buffer_from_struct; +using batt::byte_distance; using batt::ConstBuffer; using batt::make_buffer; using batt::mutable_buffer_from_struct; using batt::MutableBuffer; using batt::resize_buffer; -using llfs::advance_pointer; -using llfs::byte_distance; - } // namespace turtle_kv diff --git a/src/turtle_kv/mem_table/mem_table.test.cpp b/src/turtle_kv/mem_table/mem_table.test.cpp index 4ad2647..121ea0d 100644 --- a/src/turtle_kv/mem_table/mem_table.test.cpp +++ b/src/turtle_kv/mem_table/mem_table.test.cpp @@ -377,9 +377,9 @@ TEST_F(MemTableTest, PutGet) // TEST_F(MemTableTest, PutUntilFull) { - usize total_key_bytes = 0; - usize total_value_bytes = 0; - usize put_count = 0; + [[maybe_unused]] usize total_key_bytes = 0; + [[maybe_unused]] usize total_value_bytes = 0; + [[maybe_unused]] usize put_count = 0; for (;;) { KeyView key = this->make_random_key(); diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/packed_leaf_block.hpp new file mode 100644 index 0000000..7339cd7 --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_block.hpp @@ -0,0 +1,175 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_HPP + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline KeyView get_key(const PackedKeyValueSlotPtr& p_kv) noexcept +{ + return get_key(*p_kv); +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedLeafBlock { + static constexpr u32 kMagic = 0x7370b49full; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u32 magic; // +4 = 4 + little_u16 shared_prefix_size; // +2 = 6 + PackedKeyValueSlotPtr items_[1]; // +2 = 8 + + //+++++++++++-+-+--+----- --- -- - - - - + + static const PackedLeafBlock& view_of(const ConstBuffer& buffer) noexcept + { + BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); + + const auto* block = static_cast(buffer.data()); + + BATT_CHECK_EQ(block->magic, PackedLeafBlock::kMagic); + + return *block; + } + + //+++++++++++-+-+--+----- --- -- - - - - + + usize item_count() const noexcept + { + return this->items_end() - this->items_begin(); + } + + KeyView key_at(usize i) const noexcept + { + return this->items_[i]->key_view(); + } + + ValueView value_at(usize i) const noexcept + { + return this->items_[i]->value_view(&this->items_[i]); + } + + EditView edit_at(usize i) const noexcept + { + auto& packed = *this->items_[i]; + return EditView{packed.key_view(), packed.value_view(&this->items_[i])}; + } + + Optional item_at(usize i) const noexcept + { + return to_item_view(this->edit_at(i)); + } + + const PackedKeyValueSlotPtr& front_item() const noexcept + { + return this->items_[0]; + } + + const PackedKeyValueSlotPtr& back_item() const noexcept + { + return this->items_[this->item_count() - 1]; + } + + const PackedKeyValueSlotPtr* items_begin() const noexcept + { + return this->items_; + } + + const PackedKeyValueSlotPtr* items_end() const noexcept + { + return ((const PackedKeyValueSlotPtr*)this->items_[0].get()) - 1; + } + + Slice items_slice() const noexcept + { + return as_slice(this->items_begin(), this->items_end()); + } + + KeyView min_key() const noexcept + { + return get_key(this->front_item()); + } + + KeyView max_key() const noexcept + { + return get_key(this->back_item()); + } + + const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept + { + auto [first, last] = + std::equal_range(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; + }); + + if (first == last) { + return nullptr; + } + return std::addressof(*first); + } + + const PackedKeyValueSlotPtr* lower_bound(const KeyView& key) const noexcept + { + return std::lower_bound(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; + }); + } +}; + +static_assert(sizeof(PackedLeafBlock) == 8); + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedLeafBlockStats { + usize block_size; + usize item_count; + usize item_slot_bytes; + usize item_ptr_bytes; + + //+++++++++++-+-+--+----- --- -- - - - - + + template + static PackedLeafBlockStats from(const RangeT& src, usize dst_size) noexcept; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ()))>> +StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept; + +} // namespace turtle_kv + +#include "packed_leaf_block.ipp" diff --git a/src/turtle_kv/tree/packed_leaf_block.ipp b/src/turtle_kv/tree/packed_leaf_block.ipp new file mode 100644 index 0000000..ec4fb9c --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_block.ipp @@ -0,0 +1,100 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_IPP + +#include "packed_leaf_block.hpp" + +#include + +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +/*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, + usize dst_size) noexcept +{ + PackedLeafBlockStats stats{ + .block_size = 0, + .item_count = 0, + .item_slot_bytes = 0, + .item_ptr_bytes = 0, + }; + + if (dst_size < sizeof(PackedLeafBlock)) { + return stats; + } + stats.block_size = dst_size; + usize offset = 0; + dst_size -= sizeof(PackedLeafBlock); + offset += sizeof(PackedLeafBlock); + + for (const auto& src_item : src) { + const usize slot_size = packed_key_value_slot_size(src_item); + const usize total_item_size = slot_size + sizeof(PackedKeyValueSlotPtr); + if (dst_size < total_item_size) { + break; + } + stats.item_count += 1; + stats.item_slot_bytes += slot_size; + stats.item_ptr_bytes += sizeof(PackedKeyValueSlotPtr); + dst_size -= total_item_size; + offset += total_item_size; + + if constexpr (false) { + LOG(INFO) << BATT_INSPECT(offset) << BATT_INSPECT_STR(get_key(src_item)) + << BATT_INSPECT(stats.item_count); + } + } + + return stats; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept +{ + if (dst.size() < sizeof(PackedLeafBlock)) { + return {batt::StatusCode::kResourceExhausted}; + } + + auto stats = PackedLeafBlockStats::from(src, dst.size()); + if (stats.block_size != dst.size()) { + return {batt::StatusCode::kResourceExhausted}; + } + + PackedLeafBlock* block = static_cast(dst.data()); + { + block->magic = PackedLeafBlock::kMagic; + block->items_[0].offset = + byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes)); + } + + PackedKeyValueSlotPtr* pp_slot = block->items_; + void* p_slot = const_cast(pp_slot->get()); + + IterT src_iter = std::begin(src); + const IterT src_end = std::next(src_iter, stats.item_count); + for (; src_iter != src_end; ++src_iter) { + const usize slot_size = pack_key_value_slot(*src_iter, p_slot); + p_slot = advance_pointer(p_slot, slot_size); + ++pp_slot; + pp_slot->offset = byte_distance(pp_slot, p_slot); + } + + return {src_iter}; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.test.cpp b/src/turtle_kv/tree/packed_leaf_block.test.cpp new file mode 100644 index 0000000..2277269 --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_block.test.cpp @@ -0,0 +1,121 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// +#include + +#include +#include + +#include + +#include +#include +#include + +namespace { + +using namespace batt::int_types; + +using batt::MutableBuffer; +using batt::StableStringStore; +using batt::StatusOr; + +using turtle_kv::EditView; +using turtle_kv::KeyView; +using turtle_kv::ValueView; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +std::string_view random_str(std::default_random_engine& rng, + usize min_size, + usize max_size, + StableStringStore& strings) noexcept +{ + std::uniform_int_distribution pick_size{min_size, max_size}; + std::uniform_int_distribution pick_char{'a', 'z'}; + + const usize n = pick_size(rng); + MutableBuffer buf = strings.allocate(n); + char* chars = static_cast(buf.data()); + + for (usize i = 0; i < n; ++i, ++chars) { + *chars = pick_char(rng); + } + + return std::string_view{static_cast(buf.data()), buf.size()}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST(TreePackedLeafBlockTest, Random) +{ + const usize kNumSeeds = 1000; + const usize kMinKeySize = 4; + const usize kMaxKeySize = 48; + const usize kMinValueSize = 0; + const usize kMaxValueSize = 200; + const usize kBlockSize = 8192; + + for (usize seed = 0; seed < kNumSeeds; ++seed) { + std::default_random_engine rng{seed}; + + StableStringStore strings; + std::unordered_set used_keys; + std::vector src_edits; + + // Generate enough random edits to fill a block. + // + usize src_size = 0; + while (src_size < kBlockSize) { + std::string_view key = random_str(rng, kMinKeySize, kMaxKeySize, strings); + if (used_keys.count(key)) { + continue; + } + used_keys.insert(key); + std::string_view value = random_str(rng, kMinValueSize, kMaxValueSize, strings); + EditView edit{key, ValueView::from_str(value)}; + src_edits.push_back(edit); + src_size += key.size() + value.size(); + } + std::sort(src_edits.begin(), src_edits.end(), turtle_kv::KeyOrder{}); + + // Pack a block. + // + std::array block_buffer; + block_buffer.fill('!'); + auto dst_buffer = MutableBuffer{block_buffer.data(), kBlockSize}; + + StatusOr::const_iterator> consumed_src_end = + turtle_kv::pack_leaf_block(src_edits, dst_buffer); + + ASSERT_TRUE(consumed_src_end.ok()); + + for (usize i = kBlockSize; i < kBlockSize * 2; ++i) { + ASSERT_EQ(block_buffer[i], '!') << BATT_INSPECT(i); + } + + const usize packed_count = *consumed_src_end - src_edits.begin(); + const auto& packed_block = turtle_kv::PackedLeafBlock::view_of(dst_buffer); + + usize found_count = 0; + for (const EditView& src_edit : src_edits) { + auto* found_ptr = packed_block.find_key(get_key(src_edit)); + if (found_count < packed_count) { + ASSERT_NE(found_ptr, nullptr) + << BATT_INSPECT(src_edit) << BATT_INSPECT(found_count) << BATT_INSPECT(packed_count); + ++found_count; + } else { + ASSERT_EQ(found_ptr, nullptr); + } + } + } +} + +} // namespace diff --git a/src/turtle_kv/tree/packed_leaf_page2.hpp b/src/turtle_kv/tree/packed_leaf_page2.hpp new file mode 100644 index 0000000..22f925f --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_page2.hpp @@ -0,0 +1,37 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_PAGE2_HPP + +#include + +#include + +#include +#include + +namespace turtle_kv { + +struct PackedLeafPageHeader2 { + static constexpr u64 kMagic = 0x6456beb7f9558445ull; + + //+++++++++++-+-+--+----- --- -- - - - - + + u32 key_count; // +4 = 12 + u32 total_packed_size; // +4 = 16 + llfs::PackedPointer> items; // +4 = 20 + u8 pad_[12]; // +12 = 32 +#if 0 + u32 index_step; // +4 = 16 + u32 index_size; // +4 = 20 + llfs::PackedPointer trie_index; // +4 = 32 +#endif +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_node_page.cpp b/src/turtle_kv/tree/packed_node_page.cpp index bff419e..6dbdd78 100644 --- a/src/turtle_kv/tree/packed_node_page.cpp +++ b/src/turtle_kv/tree/packed_node_page.cpp @@ -270,8 +270,8 @@ StatusOr> PackedNodePage::create_piecewise_filter(usize lev if (filter_data.values.empty()) { // Entire segment is live. // - live_ranges.emplace_back( - Interval{PiecewiseFilter::kMinLowerBound, PiecewiseFilter::kMaxUpperBound}); + live_ranges.emplace_back(Interval{PiecewiseFilter::kMinLowerBound, + PiecewiseFilter::kMaxUpperBound}); } else { live_ranges.emplace_back( Interval{PiecewiseFilter::kMinLowerBound, filter_data.values[i].value()}); @@ -496,4 +496,4 @@ std::function PackedNodePage::dump() const }; } -} // namespace turtle_kv \ No newline at end of file +} // namespace turtle_kv From d797ed7db883672ad7da03387ae51cf22ecdf56a Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Fri, 5 Jun 2026 16:08:26 -0400 Subject: [PATCH 02/13] upgraded requirements, wip packed_blocked_leaf_page --- conanfile.py | 12 +-- cor.yml | 2 +- src/CMakeLists.txt | 1 + src/turtle_kv/core/key_range.hpp | 23 +--- src/turtle_kv/core/packed_key_value_slot.hpp | 10 ++ src/turtle_kv/mem_table/mem_table.hpp | 2 +- src/turtle_kv/mem_table/mem_table.ipp | 4 +- .../tree/packed_blocked_leaf_page.hpp | 57 ++++++++++ .../tree/packed_blocked_leaf_page.ipp | 94 ++++++++++++++++ src/turtle_kv/tree/packed_leaf_block.hpp | 61 ++++------- src/turtle_kv/tree/packed_leaf_block.ipp | 100 +++++++++++++++++- src/turtle_kv/tree/packed_leaf_block.test.cpp | 88 +++++++++++++-- src/turtle_kv/tree/packed_leaf_page2.hpp | 37 ------- src/turtle_kv/util/art.hpp | 2 +- 14 files changed, 376 insertions(+), 117 deletions(-) create mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.hpp create mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.ipp delete mode 100644 src/turtle_kv/tree/packed_leaf_page2.hpp diff --git a/conanfile.py b/conanfile.py index 6892cf6..5a1febd 100644 --- a/conanfile.py +++ b/conanfile.py @@ -16,7 +16,7 @@ class TurtleKvRecipe(ConanFile): name = "turtle_kv" - python_requires = "cor_recipe_utils/0.19.1" + python_requires = "cor_recipe_utils/0.21.4.dev2+g938c9a386" python_requires_extend = "cor_recipe_utils.ConanFileBase" settings = "os", "compiler", "build_type", "arch" @@ -84,19 +84,19 @@ def requirements(self): "force": True, } - self.requires("abseil/20250127.0", **VISIBLE, **OVERRIDE) + self.requires("abseil/[>=20260107.1]", **VISIBLE, **OVERRIDE) self.requires("artc/[>=0.0.1 <1]") self.requires("batteries/[>=0.70.2 <1]", **VISIBLE, **OVERRIDE) - self.requires("boost/1.88.0", **VISIBLE, **OVERRIDE) - self.requires("glog/0.7.1", **VISIBLE) + self.requires("boost/[>=1.88.0 <2]", **VISIBLE, **OVERRIDE) + self.requires("glog/[>=0.7.1 <1]", **VISIBLE) self.requires("llfs/[>=0.44.0 <1]", **VISIBLE) - self.requires("pcg-cpp/cci.20220409", **VISIBLE) + self.requires("pcg-cpp/[>=cci.20220409]", **VISIBLE) self.requires("yaml-cpp/[>=0.9.0 <1]") self.requires("zlib/1.3.1", **OVERRIDE) # boost/1.88.0 and ninja/1.13.2 depend (exactly) on libbacktrace/cci.20210118 # - self.requires("libbacktrace/[>=cci.20240730]", **OVERRIDE) + self.requires("libbacktrace/[>=cci.20210118]") if platform.system() == "Linux": if self.options.with_keyvcr: diff --git a/cor.yml b/cor.yml index 2a24d1b..c37142c 100644 --- a/cor.yml +++ b/cor.yml @@ -1,3 +1,3 @@ cor: cli: - version: 0.19.1 + version: 0.21.4.dev2+g938c9a386 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8631e97..ab268b2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,7 @@ endif () target_link_libraries( turtle_kv PUBLIC + artc::artc abseil::abseil batteries::batteries boost::boost diff --git a/src/turtle_kv/core/key_range.hpp b/src/turtle_kv/core/key_range.hpp index 16febe1..2e1c198 100644 --- a/src/turtle_kv/core/key_range.hpp +++ b/src/turtle_kv/core/key_range.hpp @@ -34,27 +34,12 @@ inline CInterval get_key_range(const Chunk& chunk) }; } -inline CInterval get_key_range(const EditView& edit) +template +inline CInterval get_key_range(const T& has_key_view) { return CInterval{ - .lower_bound = get_key(edit), - .upper_bound = get_key(edit), - }; -} - -inline CInterval get_key_range(const ItemView& item) -{ - return CInterval{ - .lower_bound = get_key(item), - .upper_bound = get_key(item), - }; -} - -inline CInterval get_key_range(const KeyView& key) -{ - return CInterval{ - .lower_bound = key, - .upper_bound = key, + .lower_bound = get_key(has_key_view), + .upper_bound = get_key(has_key_view), }; } diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index 0f0d2d2..ce42137 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -150,6 +150,16 @@ inline PackedKeyValueSlotRef to_key_value_slot_ref(const ConstBuffer& slot_buffe }; } +inline KeyView get_key(const PackedKeyValueSlotPtr& p_kv) noexcept +{ + return get_key(*p_kv); +} + +inline ValueView get_value(const PackedKeyValueSlotPtr& p_kv) noexcept +{ + return p_kv->value_view(std::addressof(p_kv)); +} + template concept ConvertibleToKeyValueSlotRef = requires(const T& obj) { { to_key_value_slot_ref(obj) } -> std::convertible_to; diff --git a/src/turtle_kv/mem_table/mem_table.hpp b/src/turtle_kv/mem_table/mem_table.hpp index 780c7b4..fda4452 100644 --- a/src/turtle_kv/mem_table/mem_table.hpp +++ b/src/turtle_kv/mem_table/mem_table.hpp @@ -488,7 +488,7 @@ class BasicMemTable::PerOpStorageContext // One thread will acquire a lock, others will block at this point. // - absl::MutexLock lock{&this->mem_table_.block_list_mutex_}; + absl::MutexLock lock{this->mem_table_.block_list_mutex_}; // If there are no block buffers attached to the MemTable, then we may just have to wait until // the checkpoint update pipeline catches up. If there are block buffers attached, then its diff --git a/src/turtle_kv/mem_table/mem_table.ipp b/src/turtle_kv/mem_table/mem_table.ipp index 4f5930b..2812e28 100644 --- a/src/turtle_kv/mem_table/mem_table.ipp +++ b/src/turtle_kv/mem_table/mem_table.ipp @@ -395,7 +395,7 @@ void BasicMemTable::handle_external_cache_alloc(i6 this->allocation_tracker_.allocate_external(cache_alloc_delta, overcommit); { - absl::MutexLock lock{&this->block_list_mutex_}; + absl::MutexLock lock{this->block_list_mutex_}; BATT_CHECK(this->cache_alloc_in_progress_); this->total_cache_alloc_.subsume(std::move(alloc)); this->cache_alloc_in_progress_ = false; @@ -417,7 +417,7 @@ void BasicMemTable::handle_external_cache_alloc(i6 } else if (cache_alloc_delta < 0) { StatusOr alloc_to_release; { - absl::MutexLock lock{&this->block_list_mutex_}; + absl::MutexLock lock{this->block_list_mutex_}; BATT_CHECK(this->cache_alloc_in_progress_); alloc_to_release = this->total_cache_alloc_.split(-cache_alloc_delta); this->cache_alloc_in_progress_ = false; diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp new file mode 100644 index 0000000..985f5af --- /dev/null +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp @@ -0,0 +1,57 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP + +#include + +#include + +#include + +#include +#include + +#include +#include + +namespace turtle_kv { + +struct PackedBlockedLeafPage { + static constexpr u64 kMagic = 0x6456beb7f9558445ull; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u64 magic; + little_u32 item_count; + little_u32 total_packed_size; + little_u32 blocks_per_trie_key; + little_u32 block_size_bytes; + little_u32 block_count; + little_u32 block0_byte_offset_in_page; + + /** \brief Pointer to array that stores, for each block, the starting item index relative to the + * entire leaf. + */ + llfs::PackedPointer> block_starting_item; + + /** \brief Pointer to packed ART index. + */ + llfs::PackedPointer art_block_index; + + //+++++++++++-+-+--+----- --- -- - - - - +}; + +template +StatusOr pack_blocked_leaf_page(const ItemRangeT& src_items, + MutableBuffer dst_buffer) noexcept; + +} // namespace turtle_kv + +#include "packed_blocked_leaf_page.ipp" diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp new file mode 100644 index 0000000..6ec3864 --- /dev/null +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp @@ -0,0 +1,94 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP + +#include "packed_blocked_leaf_page.hpp" + +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +StatusOr pack_blocked_leaf_page(const ItemRangeT& src_items, + MutableBuffer dst_buffer) noexcept +{ + const usize block_size = 8192; + const usize item_count = std::size(src_items); + + //+++++++++++-+-+--+----- --- -- - - - - + // Calculate the number of blocks needed and how many items in each one. + // + SmallVec block_stats; + { + auto src_iter = std::begin(src_items); + const auto src_end = std::end(src_items); + usize blocks_size_remaining = dst_buffer.size() - block_size; + for (;;) { + if (src_iter == src_end) { + break; + } + BATT_CHECK_LT(src_iter, src_end); + + BATT_ASSIGN_OK_RESULT(auto stats, + PackedLeafBlockStats::from(std::ranges::subrange(src_iter, src_end), + blocks_size_remaining)); + + blocks_size_remaining -= block_size; + src_iter = std::next(src_iter, stats.item_count); + } + } + const usize block_count = block_stats.size(); + const usize block_starting_item_array_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * block_count; + + //+++++++++++-+-+--+----- --- -- - - - - + // Initialize the leaf header. + // + MutableBuffer dst_remaining = dst_buffer; + dst_remaining += sizeof(llfs::PackedPageHeader); + + auto* leaf_header = static_cast(dst_remaining.data()); + { + leaf_header->magic = PackedBlockedLeafPage::kMagic; + leaf_header->item_count = BATT_CHECKED_CAST(u32, item_count); + leaf_header->total_packed_size = 0; // TODO [tastolfi 2026-06-01] + leaf_header->blocks_per_trie_key = 0; // TODO [tastolfi 2026-06-01] + leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); + leaf_header->block_count = BATT_CHECKED_CAST(u32, block_count); + leaf_header->block0_byte_offset_in_page = 0; // TODO [tastolfi 2026-06-01] + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack `block_starting_item` array. + // + + { + auto* block_starting_item = static_cast*>(dst_remaining.data()); + dst_remaining += block_starting_item_array_size; + + block_starting_item->initialize(block_stats.size()); + + little_u32* block_start = block_starting_item->data(); + u32 item_i = 0; + for (const PackedLeafBlockStats& stats : block_stats) { + *block_start = item_i; + item_i += stats.item_count; + ++block_start; + } + + leaf_header->block_starting_item.reset_unsafe(block_starting_item); + } +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/packed_leaf_block.hpp index 7339cd7..12f125e 100644 --- a/src/turtle_kv/tree/packed_leaf_block.hpp +++ b/src/turtle_kv/tree/packed_leaf_block.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -27,13 +28,6 @@ namespace turtle_kv { -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline KeyView get_key(const PackedKeyValueSlotPtr& p_kv) noexcept -{ - return get_key(*p_kv); -} - //=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- // struct PackedLeafBlock { @@ -47,16 +41,10 @@ struct PackedLeafBlock { //+++++++++++-+-+--+----- --- -- - - - - - static const PackedLeafBlock& view_of(const ConstBuffer& buffer) noexcept - { - BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); - - const auto* block = static_cast(buffer.data()); - - BATT_CHECK_EQ(block->magic, PackedLeafBlock::kMagic); - - return *block; - } + /** \brief Returns the passed buffer's memory region, validated as a PackedLeafBlock and cast to + * `const PackedLeafBlock &`. + */ + static const PackedLeafBlock& view_of(const ConstBuffer& buffer) noexcept; //+++++++++++-+-+--+----- --- -- - - - - @@ -111,6 +99,9 @@ struct PackedLeafBlock { return as_slice(this->items_begin(), this->items_end()); } + Slice items_slice(Optional key_lower_bound, + Optional key_upper_bound) const noexcept; + KeyView min_key() const noexcept { return get_key(this->front_item()); @@ -121,31 +112,19 @@ struct PackedLeafBlock { return get_key(this->back_item()); } - const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept + KeyView shared_key_prefix() const noexcept { - auto [first, last] = - std::equal_range(this->items_begin(), - this->items_end(), - key, - [](const auto& l, const auto& r) { - return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; - }); - - if (first == last) { - return nullptr; - } - return std::addressof(*first); + return this->min_key().substr(0, this->shared_prefix_size); } - const PackedKeyValueSlotPtr* lower_bound(const KeyView& key) const noexcept - { - return std::lower_bound(this->items_begin(), - this->items_end(), - key, - [](const auto& l, const auto& r) { - return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; - }); - } + /** \brief Returns an iterator to the given key in this block if found or nullptr if not found. + */ + const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept; + + /** \brief Returns an iterator to the first item in this block whose key is not less than `key`; + * if all keys in the block are less than `key`, returns `this->items_end()`. + */ + const PackedKeyValueSlotPtr* lower_bound(const KeyView& key) const noexcept; }; static_assert(sizeof(PackedLeafBlock) == 8); @@ -168,7 +147,9 @@ struct PackedLeafBlockStats { // template ()))>> -StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept; +StatusOr pack_leaf_block(const RangeT& src, + MutableBuffer dst, + const Optional& stats = None) noexcept; } // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.ipp b/src/turtle_kv/tree/packed_leaf_block.ipp index ec4fb9c..fe2f31f 100644 --- a/src/turtle_kv/tree/packed_leaf_block.ipp +++ b/src/turtle_kv/tree/packed_leaf_block.ipp @@ -11,8 +11,14 @@ #include "packed_leaf_block.hpp" +#include + #include +#include + +#include +#include #include #include @@ -64,13 +70,18 @@ template //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template -StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept +StatusOr pack_leaf_block(const RangeT& src, + MutableBuffer dst, + const Optional& opt_stats) noexcept { if (dst.size() < sizeof(PackedLeafBlock)) { return {batt::StatusCode::kResourceExhausted}; } - auto stats = PackedLeafBlockStats::from(src, dst.size()); + PackedLeafBlockStats stats = opt_stats.or_else([&] { + return PackedLeafBlockStats::from(src, dst.size()); + }); + if (stats.block_size != dst.size()) { return {batt::StatusCode::kResourceExhausted}; } @@ -82,6 +93,8 @@ StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes)); } + //----- --- -- - - - - + // Pack all slot data. PackedKeyValueSlotPtr* pp_slot = block->items_; void* p_slot = const_cast(pp_slot->get()); @@ -94,7 +107,90 @@ StatusOr pack_leaf_block(const RangeT& src, MutableBuffer dst) noexcept pp_slot->offset = byte_distance(pp_slot, p_slot); } + //----- --- -- - - - - + // Set the common prefix. + // + block->shared_prefix_size = + BATT_CHECKED_CAST(u16, + llfs::find_common_prefix(0, block->min_key(), block->max_key()).size()); + return {src_iter}; } +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// struct PackedLeafBlock + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/*static*/ const PackedLeafBlock& PackedLeafBlock::view_of(const ConstBuffer& buffer) noexcept +{ + BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); + + const auto* block = static_cast(buffer.data()); + + BATT_CHECK_EQ(block->magic, PackedLeafBlock::kMagic); + + return *block; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline const PackedKeyValueSlotPtr* PackedLeafBlock::find_key(const KeyView& key) const noexcept +{ + const auto convert_result = [](auto&& first_last_pair) -> const PackedKeyValueSlotPtr* { + if (first_last_pair.first == first_last_pair.second) { + return nullptr; + } + return first_last_pair.first; + }; + + if (this->shared_prefix_size > 0) { + const usize prefix_size = this->shared_prefix_size; + if (key.size() < prefix_size) { + return nullptr; + } + auto order = batt::compare(key.substr(0, prefix_size), this->shared_key_prefix()); + if (order != batt::Order::Equal) { + return nullptr; + } + + return convert_result( + std::equal_range(this->items_begin(), this->items_end(), key, KeySuffixOrder{prefix_size})); + } + + return convert_result(std::equal_range(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == + batt::Order::Less; + })); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline const PackedKeyValueSlotPtr* PackedLeafBlock::lower_bound(const KeyView& key) const noexcept +{ + return std::lower_bound(this->items_begin(), + this->items_end(), + key, + [](const auto& l, const auto& r) { + return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; + }); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +Slice PackedLeafBlock::items_slice( + Optional key_lower_bound, + Optional key_upper_bound) const noexcept +{ + auto [first, last] = std::equal_range(this->items_begin(), + this->items_end(), + Interval{key_lower_bound.or_else(global_min_key), + key_upper_bound.or_else(global_max_key)}, + ExtendedKeyRangeOrder{}); + return as_slice(first, last); +} + } // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.test.cpp b/src/turtle_kv/tree/packed_leaf_block.test.cpp index 2277269..8656778 100644 --- a/src/turtle_kv/tree/packed_leaf_block.test.cpp +++ b/src/turtle_kv/tree/packed_leaf_block.test.cpp @@ -33,18 +33,25 @@ using turtle_kv::ValueView; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // +template std::string_view random_str(std::default_random_engine& rng, + SizeDistribution&& pick_size, usize min_size, usize max_size, - StableStringStore& strings) noexcept + StableStringStore& strings, + std::string_view prefix = "") noexcept { - std::uniform_int_distribution pick_size{min_size, max_size}; std::uniform_int_distribution pick_char{'a', 'z'}; - const usize n = pick_size(rng); - MutableBuffer buf = strings.allocate(n); + const usize n = min_size + std::min(pick_size(rng), max_size - min_size); + MutableBuffer buf = strings.allocate(prefix.size() + n); char* chars = static_cast(buf.data()); + if (!prefix.empty()) { + std::memcpy(chars, prefix.data(), prefix.size()); + chars += prefix.size(); + } + for (usize i = 0; i < n; ++i, ++chars) { *chars = pick_char(rng); } @@ -57,16 +64,28 @@ std::string_view random_str(std::default_random_engine& rng, TEST(TreePackedLeafBlockTest, Random) { const usize kNumSeeds = 1000; + const usize kNumNotFoundQueries = 100; + const usize kNumLowerBoundQueries = 500; + const usize kMinPrefixSize = 0; + const usize kMaxPrefixSize = 8; const usize kMinKeySize = 4; const usize kMaxKeySize = 48; const usize kMinValueSize = 0; const usize kMaxValueSize = 200; const usize kBlockSize = 8192; + std::geometric_distribution pick_prefix_size{0.5}; + std::geometric_distribution pick_key_size{0.7}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMaxValueSize}; + for (usize seed = 0; seed < kNumSeeds; ++seed) { std::default_random_engine rng{seed}; StableStringStore strings; + + std::string_view prefix = + random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); + std::unordered_set used_keys; std::vector src_edits; @@ -74,14 +93,17 @@ TEST(TreePackedLeafBlockTest, Random) // usize src_size = 0; while (src_size < kBlockSize) { - std::string_view key = random_str(rng, kMinKeySize, kMaxKeySize, strings); + std::string_view key = + random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); if (used_keys.count(key)) { continue; } used_keys.insert(key); - std::string_view value = random_str(rng, kMinValueSize, kMaxValueSize, strings); - EditView edit{key, ValueView::from_str(value)}; - src_edits.push_back(edit); + + std::string_view value = + random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); + + src_edits.push_back(EditView{key, ValueView::from_str(value)}); src_size += key.size() + value.size(); } std::sort(src_edits.begin(), src_edits.end(), turtle_kv::KeyOrder{}); @@ -104,6 +126,8 @@ TEST(TreePackedLeafBlockTest, Random) const usize packed_count = *consumed_src_end - src_edits.begin(); const auto& packed_block = turtle_kv::PackedLeafBlock::view_of(dst_buffer); + ASSERT_EQ(packed_block.shared_prefix_size.value(), prefix.size()); + usize found_count = 0; for (const EditView& src_edit : src_edits) { auto* found_ptr = packed_block.find_key(get_key(src_edit)); @@ -111,10 +135,58 @@ TEST(TreePackedLeafBlockTest, Random) ASSERT_NE(found_ptr, nullptr) << BATT_INSPECT(src_edit) << BATT_INSPECT(found_count) << BATT_INSPECT(packed_count); ++found_count; + + ASSERT_EQ(get_key(*found_ptr), get_key(src_edit)); + ASSERT_EQ(get_value(*found_ptr), get_value(src_edit)); } else { ASSERT_EQ(found_ptr, nullptr); } } + + for (usize i = 0; i < kNumNotFoundQueries; ++i) { + std::string_view key; + for (;;) { + key = random_str(rng, + pick_key_size, + kMinKeySize + prefix.size(), + kMaxKeySize + prefix.size(), + strings); + if (!used_keys.count(key)) { + break; + } + } + + ASSERT_EQ(packed_block.find_key(key), nullptr); + } + + for (usize i = 0; i < kNumLowerBoundQueries; ++i) { + std::string_view key = (i % 2) ? random_str(rng, + pick_key_size, + kMinKeySize + prefix.size(), + kMaxKeySize + prefix.size(), + strings) + : random_str(rng, // + pick_key_size, + kMinKeySize, + kMaxKeySize, + strings, + prefix); + + const auto expected_iter = + std::lower_bound(src_edits.begin(), src_edits.end(), key, turtle_kv::KeyOrder{}); + + const usize expected_i = std::distance(src_edits.begin(), expected_iter); + + const auto actual_iter = packed_block.lower_bound(key); + + const usize actual_i = std::distance(packed_block.items_begin(), actual_iter); + + if (expected_i >= packed_count) { + ASSERT_EQ(actual_i, packed_count); + } else { + ASSERT_EQ(actual_i, expected_i); + } + } } } diff --git a/src/turtle_kv/tree/packed_leaf_page2.hpp b/src/turtle_kv/tree/packed_leaf_page2.hpp deleted file mode 100644 index 22f925f..0000000 --- a/src/turtle_kv/tree/packed_leaf_page2.hpp +++ /dev/null @@ -1,37 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_LEAF_PAGE2_HPP - -#include - -#include - -#include -#include - -namespace turtle_kv { - -struct PackedLeafPageHeader2 { - static constexpr u64 kMagic = 0x6456beb7f9558445ull; - - //+++++++++++-+-+--+----- --- -- - - - - - - u32 key_count; // +4 = 12 - u32 total_packed_size; // +4 = 16 - llfs::PackedPointer> items; // +4 = 20 - u8 pad_[12]; // +12 = 32 -#if 0 - u32 index_step; // +4 = 16 - u32 index_size; // +4 = 20 - llfs::PackedPointer trie_index; // +4 = 32 -#endif -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/util/art.hpp b/src/turtle_kv/util/art.hpp index b9903cd..d4c050b 100644 --- a/src/turtle_kv/util/art.hpp +++ b/src/turtle_kv/util/art.hpp @@ -948,7 +948,7 @@ class ARTBase ~MemoryContext() noexcept { if (this->art_) { - absl::MutexLock lock{&this->art_->mutex_}; + absl::MutexLock lock{this->art_->mutex_}; for (auto& p_ex : this->thread_extents_) { this->art_->extents_.emplace_back(std::move(p_ex)); } From 5da7474f6bd2066429b79062d474c51ec90ffe31 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Fri, 5 Jun 2026 17:05:25 -0400 Subject: [PATCH 03/13] wip packed blocked leaf page --- conan.lock | 4 +- conanfile.py | 2 +- cor.yml | 2 +- .../tree/packed_blocked_leaf_page.hpp | 2 +- .../tree/packed_blocked_leaf_page.ipp | 54 +++++++++++++++++-- 5 files changed, 56 insertions(+), 8 deletions(-) diff --git a/conan.lock b/conan.lock index 1607f64..7d61ffc 100644 --- a/conan.lock +++ b/conan.lock @@ -11,7 +11,7 @@ "glog/0.7.1", "gtest/1.17.0", "keyvcr/0.2.2", - "libbacktrace/cci.20240730", + "libbacktrace/cci.20210118", "libfuse/3.16.2", "libpfm4/4.13.0", "libunwind/1.8.1", @@ -56,4 +56,4 @@ ] }, "config_requires": [] -} \ No newline at end of file +} diff --git a/conanfile.py b/conanfile.py index 5a1febd..f2165c9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -16,7 +16,7 @@ class TurtleKvRecipe(ConanFile): name = "turtle_kv" - python_requires = "cor_recipe_utils/0.21.4.dev2+g938c9a386" + python_requires = "cor_recipe_utils/0.21.4.dev3+g0d8231b80" python_requires_extend = "cor_recipe_utils.ConanFileBase" settings = "os", "compiler", "build_type", "arch" diff --git a/cor.yml b/cor.yml index c37142c..0d48710 100644 --- a/cor.yml +++ b/cor.yml @@ -1,3 +1,3 @@ cor: cli: - version: 0.21.4.dev2+g938c9a386 + version: 0.21.4.dev3+g0d8231b80 diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp index 985f5af..2bf678e 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp @@ -31,7 +31,7 @@ struct PackedBlockedLeafPage { big_u64 magic; little_u32 item_count; little_u32 total_packed_size; - little_u32 blocks_per_trie_key; + little_u32 blocks_per_art_key; little_u32 block_size_bytes; little_u32 block_count; little_u32 block0_byte_offset_in_page; diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp index 6ec3864..dd2ebc3 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp @@ -15,6 +15,10 @@ #include +#include + +#include + namespace turtle_kv { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -62,8 +66,8 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it { leaf_header->magic = PackedBlockedLeafPage::kMagic; leaf_header->item_count = BATT_CHECKED_CAST(u32, item_count); - leaf_header->total_packed_size = 0; // TODO [tastolfi 2026-06-01] - leaf_header->blocks_per_trie_key = 0; // TODO [tastolfi 2026-06-01] + leaf_header->total_packed_size = 0; // TODO [tastolfi 2026-06-01] + leaf_header->blocks_per_art_key = 0; // TODO [tastolfi 2026-06-01] leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); leaf_header->block_count = BATT_CHECKED_CAST(u32, block_count); leaf_header->block0_byte_offset_in_page = 0; // TODO [tastolfi 2026-06-01] @@ -72,7 +76,6 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it //+++++++++++-+-+--+----- --- -- - - - - // Pack `block_starting_item` array. // - { auto* block_starting_item = static_cast*>(dst_remaining.data()); dst_remaining += block_starting_item_array_size; @@ -89,6 +92,51 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it leaf_header->block_starting_item.reset_unsafe(block_starting_item); } + + //+++++++++++-+-+--+----- --- -- - - - - + // Calculate blocks_per_art_key based on available space. + // + const usize space_for_art = (dst_remaining.size() & ~(block_size - 1)) - block_size * block_count; + SmallVec art_keys; + usize blocks_per_art_key = 1; + const llfs::PackedArray& block_starting_item = *(leaf_header->block_starting_item); + for (;;) { + art_keys.clear(); + auto items = std::begin(src_items); + for (usize block_i = blocks_per_art_key; block_i < block_count; block_i += blocks_per_art_key) { + const usize item_i = block_starting_item[block_i]; + art_keys.emplace_back(get_key(*(items + item_i))); + } + + using artc::packed::PackedARTBuilder; + + batt::StableStringStore string_store; + + BATT_ASSIGN_OK_RESULT(auto art_builder, + PackedARTBuilder::from_items(art_keys.begin(), + art_keys.end(), + BATT_OVERLOADS_OF(get_key), + string_store)); + + if (art_builder.get_packed_size() > space_for_art) { + ++blocks_per_art_key; + continue; + } + + MutableBuffer art_buffer{dst_remaining.data(), art_builder.get_packed_size()}; + dst_remaining += art_buffer.size(); + BATT_CHECK_GE(dst_remaining.size(), block_size * block_count); + + BATT_ASSIGN_OK_RESULT(const artc::packed::NodeBase* art_root, art_builder.build(art_buffer)); + + leaf_header->art_block_index.reset_unsafe(art_root); + break; + } + + // Shift the remaining buffer forward so it aligns with the nearest block boundary. + // + const usize offset_for_block_align = dst_remaining.size() & (block_size - 1); + dst_remaining += } } // namespace turtle_kv From 4572d2a55e0d005a0a47a11cc45f6b1c9e395de8 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sat, 6 Jun 2026 15:46:28 -0400 Subject: [PATCH 04/13] pack_blocked_leaf_page nominally working --- .../script/uniform_key_distribution.hpp | 8 +- .../tree/packed_blocked_leaf_page.cpp | 13 ++ .../tree/packed_blocked_leaf_page.hpp | 78 ++++++-- .../tree/packed_blocked_leaf_page.ipp | 82 +++++++-- .../tree/packed_blocked_leaf_page.test.cpp | 166 ++++++++++++++++++ src/turtle_kv/tree/packed_leaf_block.hpp | 21 ++- src/turtle_kv/tree/packed_leaf_block.ipp | 15 +- src/turtle_kv/tree/packed_leaf_block.test.cpp | 38 +--- src/turtle_kv/tree/random_str.hpp | 48 +++++ 9 files changed, 400 insertions(+), 69 deletions(-) create mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.cpp create mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp create mode 100644 src/turtle_kv/tree/random_str.hpp diff --git a/src/turtle_kv/script/uniform_key_distribution.hpp b/src/turtle_kv/script/uniform_key_distribution.hpp index 67fd211..700f1d4 100644 --- a/src/turtle_kv/script/uniform_key_distribution.hpp +++ b/src/turtle_kv/script/uniform_key_distribution.hpp @@ -37,7 +37,9 @@ inline constexpr std::array kHashSeeds = { class UniformInsertKeyDistribution : public KeyDistribution { public: - explicit UniformInsertKeyDistribution(usize key_size) noexcept : key_buffer_(key_size) + explicit UniformInsertKeyDistribution(usize key_size, usize seed = 0) noexcept + : next_ordinal_{seed} + , key_buffer_(key_size) { } @@ -48,7 +50,7 @@ class UniformInsertKeyDistribution : public KeyDistribution std::pair get_next(KeySet& inserted_keys) override { - return inserted_keys.create_key(this->format_key(this->count_.fetch_add(1))); + return inserted_keys.create_key(this->format_key(this->next_ordinal_.fetch_add(1))); } //+++++++++++-+-+--+----- --- -- - - - - @@ -77,7 +79,7 @@ class UniformInsertKeyDistribution : public KeyDistribution //+++++++++++-+-+--+----- --- -- - - - - - std::atomic count_{0}; + std::atomic next_ordinal_{0}; SmallVec key_buffer_; }; diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.cpp new file mode 100644 index 0000000..25ef37d --- /dev/null +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.cpp @@ -0,0 +1,13 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// + +namespace turtle_kv { +} diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp index 2bf678e..ed02a63 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp @@ -16,11 +16,14 @@ #include #include +#include #include #include #include +#include + namespace turtle_kv { struct PackedBlockedLeafPage { @@ -28,29 +31,82 @@ struct PackedBlockedLeafPage { //+++++++++++-+-+--+----- --- -- - - - - - big_u64 magic; - little_u32 item_count; - little_u32 total_packed_size; - little_u32 blocks_per_art_key; - little_u32 block_size_bytes; - little_u32 block_count; - little_u32 block0_byte_offset_in_page; + template + static usize packed_edit_size(const EditT& edit) noexcept + { + return PackedLeafBlock::packed_edit_size(edit); + } + + static usize estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u64 magic; // +8 -> 8 + little_u32 item_count; // +4 -> 12 + little_u32 total_packed_size; // +4 -> 16 + little_u32 blocks_per_art_key; // +4 -> 20 + little_u32 block_size_bytes; // +4 -> 24 + little_u32 block_count; // +4 -> 28 + llfs::PackedPointer block0; // +4 -> 32 /** \brief Pointer to array that stores, for each block, the starting item index relative to the * entire leaf. */ - llfs::PackedPointer> block_starting_item; + llfs::PackedPointer> block_starting_item; // +4 -> 36 /** \brief Pointer to packed ART index. */ - llfs::PackedPointer art_block_index; + llfs::PackedPointer art_block_index; // +4 -> 40 + + u8 pad_[24]; //+++++++++++-+-+--+----- --- -- - - - - }; +static_assert(sizeof(PackedBlockedLeafPage) == 64); + template -StatusOr pack_blocked_leaf_page(const ItemRangeT& src_items, - MutableBuffer dst_buffer) noexcept; +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline /*static*/ usize PackedBlockedLeafPage::estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept +{ + const usize space_after_header = + leaf_size - (sizeof(llfs::PackedPageHeader) + sizeof(PackedBlockedLeafPage)); + + const usize max_block_count = space_after_header / block_size; + + const usize block_starts_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * max_block_count; + + const usize space_after_block_starts = space_after_header - block_starts_size; + + const usize max_art_size = max_key_size * max_block_count * 2; + + const usize space_after_art = space_after_block_starts - max_art_size; + + BATT_CHECK_EQ(batt::bit_count(block_size), 1) << "Leaf block_size must be a power of 2"; + const usize space_for_blocks = space_after_art & ~(block_size - 1); + const usize block_count = space_for_blocks / block_size; + + const usize max_wasted_per_block = max_edit_size - 1; + const usize min_block_capacity = PackedLeafBlock::capacity(block_size) - max_wasted_per_block; + + const usize final_estimate = block_count * min_block_capacity; + + BATT_CHECK_GT(leaf_size, final_estimate); + + return final_estimate; +} } // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp index dd2ebc3..3d52be1 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.ipp @@ -13,8 +13,6 @@ #include -#include - #include #include @@ -24,10 +22,10 @@ namespace turtle_kv { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template -StatusOr pack_blocked_leaf_page(const ItemRangeT& src_items, - MutableBuffer dst_buffer) noexcept +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept { - const usize block_size = 8192; const usize item_count = std::size(src_items); //+++++++++++-+-+--+----- --- -- - - - - @@ -44,17 +42,18 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it } BATT_CHECK_LT(src_iter, src_end); - BATT_ASSIGN_OK_RESULT(auto stats, - PackedLeafBlockStats::from(std::ranges::subrange(src_iter, src_end), - blocks_size_remaining)); + if (blocks_size_remaining < block_size) { + return {batt::StatusCode::kResourceExhausted}; + } + + auto& stats = block_stats.emplace_back( + PackedLeafBlockStats::from(std::ranges::subrange(src_iter, src_end), block_size)); blocks_size_remaining -= block_size; src_iter = std::next(src_iter, stats.item_count); } } const usize block_count = block_stats.size(); - const usize block_starting_item_array_size = - sizeof(llfs::PackedArray) + sizeof(little_u32) * block_count; //+++++++++++-+-+--+----- --- -- - - - - // Initialize the leaf header. @@ -63,20 +62,26 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it dst_remaining += sizeof(llfs::PackedPageHeader); auto* leaf_header = static_cast(dst_remaining.data()); + dst_remaining += sizeof(PackedBlockedLeafPage); { leaf_header->magic = PackedBlockedLeafPage::kMagic; leaf_header->item_count = BATT_CHECKED_CAST(u32, item_count); - leaf_header->total_packed_size = 0; // TODO [tastolfi 2026-06-01] - leaf_header->blocks_per_art_key = 0; // TODO [tastolfi 2026-06-01] + leaf_header->total_packed_size = 0; + leaf_header->blocks_per_art_key = 0; leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); leaf_header->block_count = BATT_CHECKED_CAST(u32, block_count); - leaf_header->block0_byte_offset_in_page = 0; // TODO [tastolfi 2026-06-01] + leaf_header->block0.offset = 0; + leaf_header->block_starting_item.offset = 0; + leaf_header->art_block_index.offset = 0; } //+++++++++++-+-+--+----- --- -- - - - - // Pack `block_starting_item` array. // { + const usize block_starting_item_array_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * block_count; + auto* block_starting_item = static_cast*>(dst_remaining.data()); dst_remaining += block_starting_item_array_size; @@ -92,19 +97,20 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it leaf_header->block_starting_item.reset_unsafe(block_starting_item); } + const llfs::PackedArray& block_starting_item = *(leaf_header->block_starting_item); //+++++++++++-+-+--+----- --- -- - - - - // Calculate blocks_per_art_key based on available space. // - const usize space_for_art = (dst_remaining.size() & ~(block_size - 1)) - block_size * block_count; + const usize space_for_art = dst_remaining.size() - block_size * block_count; SmallVec art_keys; usize blocks_per_art_key = 1; - const llfs::PackedArray& block_starting_item = *(leaf_header->block_starting_item); for (;;) { art_keys.clear(); auto items = std::begin(src_items); for (usize block_i = blocks_per_art_key; block_i < block_count; block_i += blocks_per_art_key) { const usize item_i = block_starting_item[block_i]; + BATT_CHECK_LT(item_i, item_count); art_keys.emplace_back(get_key(*(items + item_i))); } @@ -112,6 +118,9 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it batt::StableStringStore string_store; + BATT_DEBUG_INFO(BATT_INSPECT_RANGE(art_keys) + << BATT_INSPECT(block_count) << BATT_INSPECT(blocks_per_art_key)); + BATT_ASSIGN_OK_RESULT(auto art_builder, PackedARTBuilder::from_items(art_keys.begin(), art_keys.end(), @@ -130,13 +139,54 @@ StatusOr pack_blocked_leaf_page(const ItemRangeT& src_it BATT_ASSIGN_OK_RESULT(const artc::packed::NodeBase* art_root, art_builder.build(art_buffer)); leaf_header->art_block_index.reset_unsafe(art_root); + leaf_header->blocks_per_art_key = BATT_CHECKED_CAST(u32, blocks_per_art_key); break; } + //+++++++++++-+-+--+----- --- -- - - - - // Shift the remaining buffer forward so it aligns with the nearest block boundary. // const usize offset_for_block_align = dst_remaining.size() & (block_size - 1); - dst_remaining += + dst_remaining += offset_for_block_align; + BATT_CHECK_LE(block_size * block_count, dst_remaining.size()); + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack the blocks. + // + leaf_header->block0.reset_unsafe(static_cast(dst_remaining.data())); + { + auto src_iter = std::begin(src_items); + const auto src_end = std::end(src_items); + usize block_i = 0; + for (const PackedLeafBlockStats& stats : block_stats) { + BATT_DEBUG_INFO(BATT_INSPECT(block_i) << BATT_INSPECT(stats)); + + BATT_CHECK_NE(src_iter, src_end); + auto src_block_items = std::ranges::subrange(src_iter, std::next(src_iter, stats.item_count)); + + BATT_CHECK_GE(dst_remaining.size(), block_size); + MutableBuffer dst_block_buffer{dst_remaining.data(), block_size}; + + auto block_end_iter = + BATT_OK_RESULT_OR_PANIC(pack_leaf_block(src_block_items, dst_block_buffer, stats)); + + BATT_CHECK_EQ(block_end_iter, std::end(src_block_items)); + + src_iter = block_end_iter; + dst_remaining += block_size; + ++block_i; + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Fill in remaining header fields. + // + leaf_header->total_packed_size = BATT_CHECKED_CAST(u32, dst_buffer.size() - dst_remaining.size()); + + //+++++++++++-+-+--+----- --- -- - - - - + // Success! (nothing succeeds like it) + // + return leaf_header; } } // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp new file mode 100644 index 0000000..1361f2c --- /dev/null +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp @@ -0,0 +1,166 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// +#include + +#include +#include + +#include "random_str.hpp" + +#include + +#include +#include + +#include +#include +#include + +namespace { + +using namespace batt::int_types; +using namespace batt::constants; + +using batt::MutableBuffer; +using batt::StableStringStore; +using batt::StatusOr; + +using turtle_kv::EditView; +using turtle_kv::KeyOrder; +using turtle_kv::KeyView; +using turtle_kv::pack_blocked_leaf_page; +using turtle_kv::PackedBlockedLeafPage; +using turtle_kv::random_str; +using turtle_kv::ValueView; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// Plan: +// 1. For different random seeds: +// - generate random set of prefixes (~10% of total keys) +// - generate keys using prefixes, with random values +// - sort +// - pack leaf; verify: +// a. all packed keys present and have right values +// b. any unpacked keys at end missing +// c. randomly generated non-present keys not found +// +TEST(TreePackedBlockedLeafPageTest, Random) +{ + const usize kNumSeeds = 1000; + const usize kLeafPageSize = 1 * kMiB; + const usize kNumPrefixes = 1000; + const usize kMinPrefixSize = 0; + const usize kMaxPrefixSize = 8; + const usize kMinKeySize = 4; + const usize kMaxKeySize = 48; + const usize kMinValueSize = 0; + const usize kMaxValueSize = 200; + const usize kBlockSize = 8192; + + BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); + + std::geometric_distribution pick_prefix_size{0.5}; + std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; + std::geometric_distribution pick_key_size{0.7}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; + + for (usize seed = 0; seed < kNumSeeds; ++seed) { + std::default_random_engine rng{seed}; + + StableStringStore strings; + + //+++++++++++-+-+--+----- --- -- - - - - + // Generate prefixes + // + std::vector prefixes; + { + std::unordered_set used_prefixes; + while (prefixes.size() < kNumPrefixes) { + std::string_view prefix = + random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); + + if (used_prefixes.count(prefix)) { + continue; + } + prefixes.push_back(prefix); + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Generate edits. + // + std::vector edits; + { + usize max_edit_size = 0; + usize max_key_size = 0; + usize total_edits_size = 0; + + std::unordered_set used_keys; + for (;;) { + std::string_view prefix = prefixes[pick_prefix(rng)]; + + std::string_view key = + random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); + + if (used_keys.count(key)) { + continue; + } + + std::string_view value = + random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); + + EditView edit{key, ValueView::from_str(value)}; + + const usize edit_size = PackedBlockedLeafPage::packed_edit_size(edit); + + const usize new_max_edit_size = std::max(max_edit_size, edit_size); + const usize new_max_key_size = std::max(max_key_size, key.size()); + + const usize space_available = PackedBlockedLeafPage::estimate_capacity(kLeafPageSize, + kBlockSize, + new_max_key_size, + new_max_edit_size); + + // Stop as soon as adding the next key would exceed the estimated space. + // + if (edit_size + total_edits_size > space_available) { + break; + } + + edits.push_back(edit); + total_edits_size += edit_size; + max_edit_size = new_max_edit_size; + max_key_size = new_max_key_size; + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Sort edits by key. + // + std::sort(edits.begin(), edits.end(), KeyOrder{}); + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack a blocked leaf page. + // + using StorageUnit = std::aligned_storage_t<4096, 4096>; + std::vector leaf_storage(kLeafPageSize / sizeof(StorageUnit)); + ASSERT_EQ(sizeof(StorageUnit) * leaf_storage.size(), kLeafPageSize); + + MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; + + StatusOr packed_leaf = + pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); + + ASSERT_TRUE(packed_leaf.ok()) << BATT_INSPECT(packed_leaf.status()); + } +} + +} // namespace diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/packed_leaf_block.hpp index 12f125e..6485e00 100644 --- a/src/turtle_kv/tree/packed_leaf_block.hpp +++ b/src/turtle_kv/tree/packed_leaf_block.hpp @@ -23,6 +23,7 @@ #include #include +#include #include @@ -41,6 +42,20 @@ struct PackedLeafBlock { //+++++++++++-+-+--+----- --- -- - - - - + template + static usize packed_edit_size(const EditT& edit) noexcept + { + const usize slot_size = packed_key_value_slot_size(edit); + const usize edit_size = slot_size + sizeof(PackedKeyValueSlotPtr); + + return edit_size; + } + + static constexpr usize capacity(usize block_size) noexcept + { + return block_size - std::min(block_size, sizeof(PackedLeafBlock)); + } + /** \brief Returns the passed buffer's memory region, validated as a PackedLeafBlock and cast to * `const PackedLeafBlock &`. */ @@ -140,9 +155,13 @@ struct PackedLeafBlockStats { //+++++++++++-+-+--+----- --- -- - - - - template - static PackedLeafBlockStats from(const RangeT& src, usize dst_size) noexcept; + static PackedLeafBlockStats from(const RangeT& src, usize block_size) noexcept; }; +BATT_OBJECT_PRINT_IMPL((inline), + PackedLeafBlockStats, + (block_size, item_count, item_slot_bytes, item_ptr_bytes)) + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template -/*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, - usize dst_size) noexcept +inline /*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, + usize dst_size) noexcept { PackedLeafBlockStats stats{ .block_size = 0, @@ -70,9 +70,9 @@ template //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template -StatusOr pack_leaf_block(const RangeT& src, - MutableBuffer dst, - const Optional& opt_stats) noexcept +inline StatusOr pack_leaf_block(const RangeT& src, + MutableBuffer dst, + const Optional& opt_stats) noexcept { if (dst.size() < sizeof(PackedLeafBlock)) { return {batt::StatusCode::kResourceExhausted}; @@ -122,7 +122,8 @@ StatusOr pack_leaf_block(const RangeT& src, //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -/*static*/ const PackedLeafBlock& PackedLeafBlock::view_of(const ConstBuffer& buffer) noexcept +inline /*static*/ const PackedLeafBlock& PackedLeafBlock::view_of( + const ConstBuffer& buffer) noexcept { BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); @@ -181,7 +182,7 @@ inline const PackedKeyValueSlotPtr* PackedLeafBlock::lower_bound(const KeyView& //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -Slice PackedLeafBlock::items_slice( +inline Slice PackedLeafBlock::items_slice( Optional key_lower_bound, Optional key_upper_bound) const noexcept { diff --git a/src/turtle_kv/tree/packed_leaf_block.test.cpp b/src/turtle_kv/tree/packed_leaf_block.test.cpp index 8656778..73a9e1f 100644 --- a/src/turtle_kv/tree/packed_leaf_block.test.cpp +++ b/src/turtle_kv/tree/packed_leaf_block.test.cpp @@ -13,6 +13,8 @@ #include #include +#include "random_str.hpp" + #include #include @@ -28,37 +30,11 @@ using batt::StableStringStore; using batt::StatusOr; using turtle_kv::EditView; +using turtle_kv::KeyOrder; using turtle_kv::KeyView; +using turtle_kv::random_str; using turtle_kv::ValueView; -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -std::string_view random_str(std::default_random_engine& rng, - SizeDistribution&& pick_size, - usize min_size, - usize max_size, - StableStringStore& strings, - std::string_view prefix = "") noexcept -{ - std::uniform_int_distribution pick_char{'a', 'z'}; - - const usize n = min_size + std::min(pick_size(rng), max_size - min_size); - MutableBuffer buf = strings.allocate(prefix.size() + n); - char* chars = static_cast(buf.data()); - - if (!prefix.empty()) { - std::memcpy(chars, prefix.data(), prefix.size()); - chars += prefix.size(); - } - - for (usize i = 0; i < n; ++i, ++chars) { - *chars = pick_char(rng); - } - - return std::string_view{static_cast(buf.data()), buf.size()}; -} - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // TEST(TreePackedLeafBlockTest, Random) @@ -76,7 +52,7 @@ TEST(TreePackedLeafBlockTest, Random) std::geometric_distribution pick_prefix_size{0.5}; std::geometric_distribution pick_key_size{0.7}; - std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMaxValueSize}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; for (usize seed = 0; seed < kNumSeeds; ++seed) { std::default_random_engine rng{seed}; @@ -106,7 +82,7 @@ TEST(TreePackedLeafBlockTest, Random) src_edits.push_back(EditView{key, ValueView::from_str(value)}); src_size += key.size() + value.size(); } - std::sort(src_edits.begin(), src_edits.end(), turtle_kv::KeyOrder{}); + std::sort(src_edits.begin(), src_edits.end(), KeyOrder{}); // Pack a block. // @@ -173,7 +149,7 @@ TEST(TreePackedLeafBlockTest, Random) prefix); const auto expected_iter = - std::lower_bound(src_edits.begin(), src_edits.end(), key, turtle_kv::KeyOrder{}); + std::lower_bound(src_edits.begin(), src_edits.end(), key, KeyOrder{}); const usize expected_i = std::distance(src_edits.begin(), expected_iter); diff --git a/src/turtle_kv/tree/random_str.hpp b/src/turtle_kv/tree/random_str.hpp new file mode 100644 index 0000000..5145d56 --- /dev/null +++ b/src/turtle_kv/tree/random_str.hpp @@ -0,0 +1,48 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include + +#include + +#include +#include +#include +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +std::string_view random_str(std::default_random_engine& rng, + SizeDistribution&& pick_size, + usize min_size, + usize max_size, + batt::StableStringStore& strings, + std::string_view prefix = "") noexcept +{ + std::uniform_int_distribution pick_char{'a', 'z'}; + + const usize n = min_size + std::min(pick_size(rng), max_size - min_size); + batt::MutableBuffer buf = strings.allocate(prefix.size() + n); + char* chars = static_cast(buf.data()); + + if (!prefix.empty()) { + std::memcpy(chars, prefix.data(), prefix.size()); + chars += prefix.size(); + } + + for (usize i = 0; i < n; ++i, ++chars) { + *chars = pick_char(rng); + } + + return std::string_view{static_cast(buf.data()), buf.size()}; +} + +} // namespace turtle_kv From 203fb60d59551df995a82380e2e6859cc845f483 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 7 Jun 2026 12:24:25 -0400 Subject: [PATCH 05/13] Add item seq to PackedBlockedLeafPage. --- .../tree/packed_blocked_leaf_page.hpp | 32 +++++++++ .../tree/packed_blocked_leaf_page.test.cpp | 30 +++++++- src/turtle_kv/tree/packed_leaf_block.hpp | 68 +++++++++++++++++++ src/turtle_kv/tree/packed_leaf_block.ipp | 10 ++- 4 files changed, 136 insertions(+), 4 deletions(-) diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp index ed02a63..5aa3008 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp @@ -23,6 +23,9 @@ #include #include +#include + +#include namespace turtle_kv { @@ -64,6 +67,35 @@ struct PackedBlockedLeafPage { u8 pad_[24]; //+++++++++++-+-+--+----- --- -- - - - - + + PackedLeafBlock::Iterator blocks_begin() const noexcept + { + return PackedLeafBlock::Iterator{this->block0.get(), (isize)this->block_size_bytes.value()}; + } + + PackedLeafBlock::Iterator blocks_end() const noexcept + { + return this->blocks_begin() + this->block_count; + } + + auto blocks() const noexcept + { + return std::ranges::subrange(this->blocks_begin(), + this->blocks_end()); + } + + auto blocks_seq() const noexcept + { + return batt::as_seq(this->blocks()); + } + + auto items_seq() const noexcept + { + return this->blocks_seq() | batt::seq::map([](const PackedLeafBlock& block) { + return batt::as_seq(block.items_slice()); + }) | + batt::seq::flatten(); + } }; static_assert(sizeof(PackedBlockedLeafPage) == 64); diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp index 1361f2c..ec26d26 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp @@ -36,6 +36,7 @@ using batt::StatusOr; using turtle_kv::EditView; using turtle_kv::KeyOrder; using turtle_kv::KeyView; +using turtle_kv::Optional; using turtle_kv::pack_blocked_leaf_page; using turtle_kv::PackedBlockedLeafPage; using turtle_kv::random_str; @@ -72,6 +73,9 @@ TEST(TreePackedBlockedLeafPageTest, Random) std::geometric_distribution pick_key_size{0.7}; std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; + usize total_keys = 0; + usize total_bytes = 0; + for (usize seed = 0; seed < kNumSeeds; ++seed) { std::default_random_engine rng{seed}; @@ -135,6 +139,9 @@ TEST(TreePackedBlockedLeafPageTest, Random) break; } + ++total_keys; + total_bytes += edit_size; + edits.push_back(edit); total_edits_size += edit_size; max_edit_size = new_max_edit_size; @@ -156,11 +163,30 @@ TEST(TreePackedBlockedLeafPageTest, Random) MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; - StatusOr packed_leaf = + StatusOr status_or_packed_leaf = pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); - ASSERT_TRUE(packed_leaf.ok()) << BATT_INSPECT(packed_leaf.status()); + ASSERT_TRUE(status_or_packed_leaf.ok()) << BATT_INSPECT(status_or_packed_leaf.status()); + + const PackedBlockedLeafPage& packed_leaf = **status_or_packed_leaf; + + //+++++++++++-+-+--+----- --- -- - - - - + // + // + { + auto packed_items = packed_leaf.items_seq(); + using Item = decltype(*packed_items.peek()); + for (const EditView& edit : edits) { + Optional next_packed = packed_items.next(); + + ASSERT_TRUE(next_packed.has_value()); + ASSERT_EQ(get_key(*next_packed), get_key(edit)); + ASSERT_EQ(get_value(*next_packed), get_value(edit)); + } + } } + + std::cerr << BATT_INSPECT(total_keys) << BATT_INSPECT(total_bytes) << std::endl; } } // namespace diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/packed_leaf_block.hpp index 6485e00..78b55ad 100644 --- a/src/turtle_kv/tree/packed_leaf_block.hpp +++ b/src/turtle_kv/tree/packed_leaf_block.hpp @@ -24,6 +24,9 @@ #include #include +#include + +#include #include @@ -36,6 +39,10 @@ struct PackedLeafBlock { //+++++++++++-+-+--+----- --- -- - - - - + class Iterator; + + //+++++++++++-+-+--+----- --- -- - - - - + big_u32 magic; // +4 = 4 little_u16 shared_prefix_size; // +2 = 6 PackedKeyValueSlotPtr items_[1]; // +2 = 8 @@ -144,6 +151,67 @@ struct PackedLeafBlock { static_assert(sizeof(PackedLeafBlock) == 8); +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +class PackedLeafBlock::Iterator + : public boost::iterator_facade< // + PackedLeafBlock::Iterator, // <- Derived + const PackedLeafBlock, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + const PackedLeafBlock&, // <- Reference + isize // <- Difference + > +{ + public: + using Self = Iterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = const PackedLeafBlock; + using reference = const PackedLeafBlock&; + + Iterator() = default; + + explicit Iterator(const PackedLeafBlock* block, isize block_size) noexcept + : block_{block} + , block_size_{block_size} + { + } + + reference dereference() const + { + return *this->block_; + } + + bool equal(const Self& other) const + { + return this->block_ == other.block_ && this->block_size_ == other.block_size_; + } + + void increment() + { + this->advance(1); + } + + void decrement() + { + this->advance(-1); + } + + void advance(isize delta) + { + this->block_ = static_cast( + advance_pointer(this->block_, delta * this->block_size_)); + } + + isize distance_to(const Self& other) const + { + return (byte_distance(this->block_, other.block_)) / this->block_size_; + } + + private: + const PackedLeafBlock* block_ = nullptr; + isize block_size_ = 0; +}; + //=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- // struct PackedLeafBlockStats { diff --git a/src/turtle_kv/tree/packed_leaf_block.ipp b/src/turtle_kv/tree/packed_leaf_block.ipp index d418c82..10c90fb 100644 --- a/src/turtle_kv/tree/packed_leaf_block.ipp +++ b/src/turtle_kv/tree/packed_leaf_block.ipp @@ -89,22 +89,28 @@ inline StatusOr pack_leaf_block(const RangeT& src, PackedLeafBlock* block = static_cast(dst.data()); { block->magic = PackedLeafBlock::kMagic; - block->items_[0].offset = - byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes)); + block->items_[0].offset = BATT_CHECKED_CAST( + u32, + byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes))); } //----- --- -- - - - - // Pack all slot data. + // PackedKeyValueSlotPtr* pp_slot = block->items_; void* p_slot = const_cast(pp_slot->get()); + void* const dst_end = advance_pointer(dst.data(), dst.size()); IterT src_iter = std::begin(src); const IterT src_end = std::next(src_iter, stats.item_count); for (; src_iter != src_end; ++src_iter) { const usize slot_size = pack_key_value_slot(*src_iter, p_slot); p_slot = advance_pointer(p_slot, slot_size); + BATT_CHECK_LE(p_slot, dst_end); ++pp_slot; pp_slot->offset = byte_distance(pp_slot, p_slot); + + BATT_CHECK_EQ((void*)pp_slot->get(), (void*)p_slot); } //----- --- -- - - - - From 511d2fe3f0df2d8ebdfbe5083b11325e1d6cbb0e Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Wed, 24 Jun 2026 10:45:37 -0400 Subject: [PATCH 06/13] Refactor. --- src/turtle_kv/core/packed_key_value_slot.hpp | 5 + .../core/packed_key_value_slot_slice.hpp | 36 ++ src/turtle_kv/tree/in_memory_node.test.cpp | 11 +- .../tree/in_memory_node_segmented_level.cpp | 2 + .../leaf/blocked_leaf_page_loader.concept.hpp | 30 ++ .../tree/leaf/packed_blocked_leaf_page.cpp | 49 +++ .../tree/leaf/packed_blocked_leaf_page.hpp | 337 ++++++++++++++++ .../{ => leaf}/packed_blocked_leaf_page.ipp | 138 ++++++- ...packed_blocked_leaf_page.item_iterator.hpp | 183 +++++++++ ..._blocked_leaf_page.sharded_live_ranges.hpp | 56 +++ ..._blocked_leaf_page.sharded_live_ranges.ipp | 170 ++++++++ .../leaf/packed_blocked_leaf_page.test.cpp | 366 ++++++++++++++++++ .../tree/{ => leaf}/packed_leaf_block.hpp | 91 +---- .../tree/{ => leaf}/packed_leaf_block.ipp | 43 +- .../tree/leaf/packed_leaf_block.iterator.hpp | 104 +++++ .../{ => leaf}/packed_leaf_block.test.cpp | 10 +- .../tree/leaf/packed_leaf_block_stats.hpp | 38 ++ .../tree/leaf/packed_leaf_block_stats.ipp | 56 +++ .../tree/packed_blocked_leaf_page.cpp | 13 - .../tree/packed_blocked_leaf_page.hpp | 145 ------- .../tree/packed_blocked_leaf_page.test.cpp | 192 --------- .../tree/packed_leaf_block_scanner.hpp | 75 ++++ src/turtle_kv/tree/packed_node_page.cpp | 9 + src/turtle_kv/tree/testing/fake_segment.hpp | 11 +- .../util/packed_piecewise_filter_view.hpp | 305 +++++++++++++++ src/turtle_kv/util/piecewise_filter.hpp | 48 ++- src/turtle_kv/util/piecewise_filter.ipp | 132 ++++--- .../util/piecewise_filter.live_subranges.hpp | 88 +++++ src/turtle_kv/util/piecewise_filter.test.cpp | 75 +++- ...piecewise_filter_storage_model.concept.hpp | 60 +++ 30 files changed, 2316 insertions(+), 562 deletions(-) create mode 100644 src/turtle_kv/core/packed_key_value_slot_slice.hpp create mode 100644 src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp rename src/turtle_kv/tree/{ => leaf}/packed_blocked_leaf_page.ipp (59%) create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp create mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp rename src/turtle_kv/tree/{ => leaf}/packed_leaf_block.hpp (68%) rename src/turtle_kv/tree/{ => leaf}/packed_leaf_block.ipp (81%) create mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp rename src/turtle_kv/tree/{ => leaf}/packed_leaf_block.test.cpp (96%) create mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp create mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp delete mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.cpp delete mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.hpp delete mode 100644 src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp create mode 100644 src/turtle_kv/tree/packed_leaf_block_scanner.hpp create mode 100644 src/turtle_kv/util/packed_piecewise_filter_view.hpp create mode 100644 src/turtle_kv/util/piecewise_filter.live_subranges.hpp create mode 100644 src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index ce42137..342f315 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -142,6 +142,11 @@ inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr* }; } +inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr& p_slot_ref) noexcept +{ + return to_key_value_slot_ref(std::addressof(p_slot_ref)); +} + inline PackedKeyValueSlotRef to_key_value_slot_ref(const ConstBuffer& slot_buffer) noexcept { return PackedKeyValueSlotRef{ diff --git a/src/turtle_kv/core/packed_key_value_slot_slice.hpp b/src/turtle_kv/core/packed_key_value_slot_slice.hpp new file mode 100644 index 0000000..7f2ee39 --- /dev/null +++ b/src/turtle_kv/core/packed_key_value_slot_slice.hpp @@ -0,0 +1,36 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_CORE_PACKED_KEY_VALUE_SLOT_SLICE_HPP + +#include + +#include + +#include + +namespace turtle_kv { + +using PackedKeyValueSlotSlice = std::variant< // + Slice, + Slice>; + +struct ToPackedKeyValueSlotSlice { + PackedKeyValueSlotSlice operator()(const Slice& ref_slice) + { + return PackedKeyValueSlotSlice{ref_slice}; + } + + PackedKeyValueSlotSlice operator()(const Slice& ptr_slice) + { + return PackedKeyValueSlotSlice{ptr_slice}; + } +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/in_memory_node.test.cpp b/src/turtle_kv/tree/in_memory_node.test.cpp index 22b3d56..52a5dd9 100644 --- a/src/turtle_kv/tree/in_memory_node.test.cpp +++ b/src/turtle_kv/tree/in_memory_node.test.cpp @@ -1,3 +1,11 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #include // #include @@ -12,9 +20,10 @@ #include #include +#include #include -#include +#include #include #include diff --git a/src/turtle_kv/tree/in_memory_node_segmented_level.cpp b/src/turtle_kv/tree/in_memory_node_segmented_level.cpp index bba1672..19edf8d 100644 --- a/src/turtle_kv/tree/in_memory_node_segmented_level.cpp +++ b/src/turtle_kv/tree/in_memory_node_segmented_level.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include #include diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp new file mode 100644 index 0000000..f9832cf --- /dev/null +++ b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp @@ -0,0 +1,30 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_BLOCKED_LEAF_PAGE_LOADER_CONCEPT_HPP + +#include + +#include + +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +template +concept BlockedLeafPageLoader = requires(T& loader, llfs::PageId page_id, BlockIndex block_i) { + loader.release_block(page_id, block_i); + { loader.load_block(page_id, block_i) } -> std::convertible_to>; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp new file mode 100644 index 0000000..5e61632 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp @@ -0,0 +1,49 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include "packed_blocked_leaf_page.hpp" +// + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/*static*/ usize PackedBlockedLeafPage::estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept +{ + const usize space_after_header = + leaf_size - (sizeof(llfs::PackedPageHeader) + sizeof(PackedBlockedLeafPage)); + + const usize max_block_count = space_after_header / block_size; + + const usize block_starts_size = + sizeof(llfs::PackedArray) + sizeof(little_u32) * max_block_count; + + const usize space_after_block_starts = space_after_header - block_starts_size; + + const usize max_art_size = max_key_size * max_block_count * 2; + + const usize space_after_art = space_after_block_starts - max_art_size; + + BATT_CHECK_EQ(batt::bit_count(block_size), 1) << "Leaf block_size must be a power of 2"; + const usize space_for_blocks = space_after_art & ~(block_size - 1); + const usize block_count = space_for_blocks / block_size; + + const usize max_wasted_per_block = max_edit_size - 1; + const usize min_block_capacity = PackedLeafBlock::capacity(block_size) - max_wasted_per_block; + + const usize final_estimate = block_count * min_block_capacity; + + BATT_CHECK_GT(leaf_size, final_estimate); + + return final_estimate; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp new file mode 100644 index 0000000..b043193 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp @@ -0,0 +1,337 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP + +#include "packed_leaf_block.hpp" +#include "packed_leaf_block.iterator.hpp" + +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include + +#include + +namespace turtle_kv { + +// Forward-declaration. +// +struct PackedBlockedLeafPage; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Packs a blocked leaf page with the passed block size, containing the passed key/value + * pairs, into the passed buffer. + */ +template +StatusOr pack_blocked_leaf_page(const usize block_size, + const ItemRangeT& src_items, + const MutableBuffer& dst_buffer) noexcept; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Header for a packed leaf page with blocked structure. + */ +struct PackedBlockedLeafPage // +{ + /** \brief Must be the first 8 bytes of the header. \see PackedBlockedLeagPage::magic + */ + static constexpr u64 kMagic = 0x6456beb7f9558445ull; + + //+++++++++++-+-+--+----- --- -- - - - - + + using BlockIterator = PackedLeafBlock::Iterator; + class ItemIterator; + + using BlockItemsSeq = PackedLeafBlock::BlockItemsSeq; + + struct ItemsSeqFromBlock { + BlockItemsSeq operator()(const PackedLeafBlock& block) const + { + return block.items_seq(); + } + }; + + using BlocksSeq = batt::SubRangeSeq>; + using ItemsSeq = batt::seq::Flatten>; + + struct SlotSliceFromBlock { + PackedKeyValueSlotSlice operator()(const PackedLeafBlock& block) const + { + return {block.items_slice()}; + } + }; + + using SlotSliceSeq = batt::seq::Map; + + template FilterModelT> + class ShardedLiveRanges; + + class HeaderShardView; + + //+++++++++++-+-+--+----- --- -- - - - - + + template + static usize packed_edit_size(const EditT& edit) noexcept + { + return PackedLeafBlock::packed_edit_size(edit); + } + + static usize estimate_capacity(usize leaf_size, + usize block_size, + usize max_key_size, + usize max_edit_size) noexcept; + + /** \brief Returns the passed buffer's memory region, validated as a PackedBlockedLeafPage and + * cast to `const PackedBlockedLeafPage &`. + */ + static const PackedBlockedLeafPage& view_of(const ConstBuffer& buffer) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + big_u64 magic; // +8 -> 8 + little_u32 total_packed_size; // +4 -> 12 + little_u32 blocks_per_art_key; // +4 -> 16 + little_u32 block_size_bytes; // +4 -> 20 + llfs::PackedPointer block0; // +4 -> 24 + + /** \brief Pointer to array that stores, for each block, the starting item index relative to the + * entire leaf. + */ + llfs::PackedPointer> block_starting_item; // +4 -> 28 + + /** \brief Pointer to packed ART index. + */ + llfs::PackedPointer art_block_index; // +4 -> 32 + + //+++++++++++-+-+--+----- --- -- - - - - + + llfs::PageId page_id() const noexcept + { + return (reinterpret_cast(this) - 1)->page_id.unpack(); + } + + Optional page_shard_id_for_block(llfs::PageCache& page_cache, + usize i, + llfs::PageId leaf_page_id) const noexcept + { + const usize block_begin_offset = this->block_page_offset(i); + const usize block_end_offset = block_begin_offset + this->block_size_bytes; + + return page_cache.page_shard_id_for(leaf_page_id, + Interval{block_begin_offset, block_end_offset}); + } + + Optional page_shard_id_for_block(llfs::PageCache& page_cache, + usize i) const noexcept + { + return this->page_shard_id_for_block(page_cache, i, this->page_id()); + } + + usize min_header_shard_size() const noexcept + { + return this->block_page_offset(0); + } + + //----- --- -- - - - - + + usize block_page_offset(usize i) const noexcept + { + return sizeof(llfs::PackedPageHeader) + offsetof(PackedBlockedLeafPage, block0) + + this->block0.offset + i * this->block_size_bytes; + } + + usize block_count() const noexcept + { + return this->block_starting_item->size() - 1; + } + + BlockIterator blocks_begin() const noexcept + { + return BlockIterator{this->block0.get(), (isize)this->block_size_bytes.value()}; + } + + const PackedLeafBlock& blocks_front() const + { + return *this->blocks_begin(); + } + + BlockIterator blocks_end() const noexcept + { + return this->blocks_begin() + this->block_count(); + } + + const PackedLeafBlock& blocks_back() const + { + return *(this->blocks_begin() + (this->block_count() - 1)); + } + + auto blocks() const noexcept + { + return std::ranges::subrange(this->blocks_begin(), this->blocks_end()); + } + + const PackedLeafBlock& block_at(usize block_i) const noexcept + { + return *(this->blocks_begin() + block_i); + } + + BlocksSeq blocks_seq() const noexcept + { + return batt::as_seq(this->blocks()); + } + + Interval item_index_range_of_block(usize i) const noexcept + { + return Interval{ + (*this->block_starting_item)[i].value(), + (*this->block_starting_item)[i + 1].value(), + }; + } + + /** \brief Returns the index of the block that would contain the given key, if it is present in + * this page. + * + * Always returns a valid block index (i.e., less-than this->block_count()) + */ + usize find_block_index_containing_key(const KeyView& key) const noexcept; + + /** \brief Returns a block iterator to the block that would contain the given key, if it is + * present in this page. + * + * Always returns a valid block iterator. + */ + BlockIterator find_block_containing_key(const KeyView& key) const noexcept; + + //----- --- -- - - - - + + /** \brief Returns the number of key/value pairs in this page. + */ + usize item_count() const noexcept + { + return this->block_starting_item->back(); + } + + /** \brief Returns a sequence of all items in the page, in key order. + */ + ItemsSeq items_seq() const noexcept + { + return this->blocks_seq() | batt::seq::map(ItemsSeqFromBlock{}) | batt::seq::flatten(); + } + + /** \brief Returns an item iterator to the first item in the page. + */ + ItemIterator items_begin() const noexcept; + + /** \brief Returns an item iterator one-past the last item in the page. + */ + ItemIterator items_end() const noexcept; + + /** \brief Returns an item iterator to the i-th item in the page. + */ + ItemIterator item_at(usize i) const noexcept; + + /** \brief Returns an iterator to the given key in this page if found or nullptr if not found. + */ + const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept; + + /** \brief Returns an iterator to the first item in this page whose key is not less than `key`; + * if all keys in the page are less than `key`, returns `this->items_end()`. + */ + ItemIterator lower_bound(const KeyView& key) const noexcept; + + //----- --- -- - - - - + + KeyView min_key() const noexcept + { + return this->blocks_front().min_key(); + } + + KeyView max_key() const noexcept + { + return this->blocks_back().max_key(); + } + + SlotSliceSeq slot_slice_seq() const noexcept + { + return this->blocks_seq() | batt::seq::map(SlotSliceFromBlock{}); + } + + template FilterModelT> + ShardedLiveRanges sharded_live_ranges( + const BasicPiecewiseFilter& filter, + const Interval& subrange) const noexcept; +}; + +static_assert(sizeof(PackedBlockedLeafPage) == 32); + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief A view of the header prefix of a PackedBlockedLeafPage. + */ +class PackedBlockedLeafPage::HeaderShardView +{ + public: + using Self = HeaderShardView; + + //+++++++++++-+-+--+----- --- -- - - - - + + static Self view_of(const ConstBuffer& buffer) noexcept + { + const PackedBlockedLeafPage& leaf = PackedBlockedLeafPage::view_of(buffer); + BATT_CHECK_GE(buffer.size(), leaf.min_header_shard_size()); + + return Self{leaf, buffer.size()}; + } + + //+++++++++++-+-+--+----- --- -- - - - - + +#if 0 + Seq load_slices(llfs::PageLoader& loader, + Optional first_key, + Optional last_key, + Optional first_index, + Optional last_index, + const PiecewiseFilter& filter); +#endif + + //+++++++++++-+-+--+----- --- -- - - - - + private: + explicit HeaderShardView(const PackedBlockedLeafPage& leaf, usize header_shard_size) noexcept + : leaf_{&leaf} + , header_shard_size_{header_shard_size} + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + const PackedBlockedLeafPage* leaf_; + usize header_shard_size_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp similarity index 59% rename from src/turtle_kv/tree/packed_blocked_leaf_page.ipp rename to src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp index 3d52be1..9d415f0 100644 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.ipp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp @@ -7,13 +7,15 @@ //+++++++++++-+-+--+----- --- -- - - - - #pragma once -#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP +#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_IPP #include "packed_blocked_leaf_page.hpp" +#include "packed_blocked_leaf_page.item_iterator.hpp" #include #include +#include #include @@ -65,11 +67,9 @@ StatusOr pack_blocked_leaf_page(const usize block_size, dst_remaining += sizeof(PackedBlockedLeafPage); { leaf_header->magic = PackedBlockedLeafPage::kMagic; - leaf_header->item_count = BATT_CHECKED_CAST(u32, item_count); leaf_header->total_packed_size = 0; leaf_header->blocks_per_art_key = 0; leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); - leaf_header->block_count = BATT_CHECKED_CAST(u32, block_count); leaf_header->block0.offset = 0; leaf_header->block_starting_item.offset = 0; leaf_header->art_block_index.offset = 0; @@ -80,12 +80,12 @@ StatusOr pack_blocked_leaf_page(const usize block_size, // { const usize block_starting_item_array_size = - sizeof(llfs::PackedArray) + sizeof(little_u32) * block_count; + sizeof(llfs::PackedArray) + sizeof(little_u32) * (block_count + 1); auto* block_starting_item = static_cast*>(dst_remaining.data()); dst_remaining += block_starting_item_array_size; - block_starting_item->initialize(block_stats.size()); + block_starting_item->initialize(block_count + 1); little_u32* block_start = block_starting_item->data(); u32 item_i = 0; @@ -94,6 +94,7 @@ StatusOr pack_blocked_leaf_page(const usize block_size, item_i += stats.item_count; ++block_start; } + *block_start = item_count; leaf_header->block_starting_item.reset_unsafe(block_starting_item); } @@ -105,13 +106,25 @@ StatusOr pack_blocked_leaf_page(const usize block_size, const usize space_for_art = dst_remaining.size() - block_size * block_count; SmallVec art_keys; usize blocks_per_art_key = 1; + //----- --- -- - - - - + const auto items = std::begin(src_items); + const auto key_at = [&items](usize i) { + return get_key(*(items + i)); + }; + //----- --- -- - - - - for (;;) { art_keys.clear(); - auto items = std::begin(src_items); for (usize block_i = blocks_per_art_key; block_i < block_count; block_i += blocks_per_art_key) { const usize item_i = block_starting_item[block_i]; BATT_CHECK_LT(item_i, item_count); - art_keys.emplace_back(get_key(*(items + item_i))); + BATT_CHECK_GT(item_i, 0); + + KeyView k0 = key_at(item_i - 1); + KeyView k1 = key_at(item_i); + KeyView common_prefix = llfs::find_common_prefix(0, k0, k1); + KeyView min_k1{k1.data(), common_prefix.size() + 1}; + + art_keys.emplace_back(min_k1); } using artc::packed::PackedARTBuilder; @@ -189,4 +202,115 @@ StatusOr pack_blocked_leaf_page(const usize block_size, return leaf_header; } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/*static*/ const PackedBlockedLeafPage& PackedBlockedLeafPage::view_of( + const ConstBuffer& buffer) noexcept +{ + BATT_CHECK_GT(buffer.size(), sizeof(PackedBlockedLeafPage) + sizeof(llfs::PackedPageHeader)); + + auto* packed = static_cast( + advance_pointer(buffer.data(), sizeof(llfs::PackedPageHeader))); + + BATT_CHECK_EQ(packed->magic, PackedBlockedLeafPage::kMagic); + + return *packed; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::items_begin() const noexcept +{ + auto first_block = this->blocks_begin(); + return ItemIterator{first_block, first_block->items_begin()}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::items_end() const noexcept +{ + auto last_block = this->blocks_end(); + return ItemIterator{last_block, last_block->items_begin()}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::item_at(usize i) const noexcept +{ + const llfs::PackedArray& starts = *this->block_starting_item; + + BATT_CHECK_NE(starts.size(), 0); + BATT_CHECK_EQ(starts.front(), 0); + + const auto iter = std::prev(std::upper_bound(starts.begin(), starts.end(), i)); + const isize item_pos_in_block = i - *iter; + const isize block_i = std::distance(starts.begin(), iter); + auto block_iter = this->blocks_begin() + block_i; + + return ItemIterator{block_iter, block_iter->items_begin() + item_pos_in_block}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline usize PackedBlockedLeafPage::find_block_index_containing_key( + const KeyView& key) const noexcept +{ + using artc::packed::find_lower_bound_rank; + using artc::packed::LowerBoundRank; + + LowerBoundRank result = find_lower_bound_rank(this->art_block_index.get(), key); + + const usize part_i = result.exact ? (result.rank + 1) : result.rank; + const usize block_i = part_i * this->blocks_per_art_key; + + BATT_CHECK_LT(block_i, this->block_count()); + + return block_i; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedLeafBlock::Iterator PackedBlockedLeafPage::find_block_containing_key( + const KeyView& key) const noexcept +{ + return this->blocks_begin() + this->find_block_index_containing_key(key); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline const PackedKeyValueSlotPtr* PackedBlockedLeafPage::find_key( + const KeyView& key) const noexcept +{ + return this->find_block_containing_key(key)->find_key(key); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::lower_bound( + const KeyView& key) const noexcept +{ + auto block_iter = this->find_block_containing_key(key); + + const PackedKeyValueSlotPtr* p_slot = block_iter->lower_bound(key); + if (p_slot == block_iter->items_end()) { + ++block_iter; + return ItemIterator{block_iter, block_iter->items_begin()}; + } + + return ItemIterator{block_iter, p_slot}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline PackedBlockedLeafPage::ShardedLiveRanges +PackedBlockedLeafPage::sharded_live_ranges(const BasicPiecewiseFilter& filter, + const Interval& subrange) const noexcept +{ + return ShardedLiveRanges{ + this->block_starting_item.get(), + filter.live_subranges_of(subrange), + }; +} + } // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp new file mode 100644 index 0000000..664adc6 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp @@ -0,0 +1,183 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_PACKED_BLOCK_LEAF_PAGE_ITEM_ITERATOR_HPP + +#include "packed_blocked_leaf_page.hpp" + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Iterator over the items in a blocked leaf page. + */ +class PackedBlockedLeafPage::ItemIterator + : public boost::iterator_facade< // + PackedBlockedLeafPage::ItemIterator, // <- Derived + const PackedKeyValueSlotPtr, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + const PackedKeyValueSlotPtr&, // <- Reference + isize // <- Difference + > +{ + public: + using Self = ItemIterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = const PackedKeyValueSlotPtr; + using reference = value_type&; + + ItemIterator() = default; + + explicit ItemIterator(BlockIterator block_iter, const PackedKeyValueSlotPtr* slot) noexcept + : block_iter_{block_iter} + , slot_{slot} + { + } + + reference dereference() const + { + return *this->slot_; + } + + bool equal(const Self& other) const + { + return this->block_iter_ == other.block_iter_ && this->slot_ == other.slot_; + } + + void increment() + { + ++this->slot_; + if (this->slot_ == this->block_iter_->items_end()) { + ++this->block_iter_; + this->slot_ = this->block_iter_->items_begin(); + } + } + + void decrement() + { + if (this->slot_ == this->block_iter_->items_begin()) { + --this->block_iter_; + this->slot_ = std::prev(this->block_iter_->items_end()); + } else { + --this->slot_; + } + } + + void advance(isize delta) + { + if (delta == 0) { + return; + } + + isize pos_in_block = this->get_item_pos_in_block(); + + if (delta > 0) { + // Keep stepping through the page one block at a time until we reduce delta to zero. + // + while (delta != 0) { + // Figure out where the current slot is in the current block. + // + const isize remaining_in_block = this->get_remaining_in_block(pos_in_block); + BATT_CHECK_GT(remaining_in_block, 0); + + // If the remaining delta is inside the block, advance the slot pointer and we are done! + // + if (delta < remaining_in_block) { + this->slot_ += delta; + break; + } + // Else reduce delta by the number of slots after this one in the current block. + // + delta -= remaining_in_block; + + // Advance to the next block, resetting the slot pointer. + // + ++this->block_iter_; + this->slot_ = this->block_iter_->items_begin(); + pos_in_block = 0; + } + + } else { // delta < 0 + + delta = -delta; + while (delta != 0) { + BATT_CHECK_GE(pos_in_block, 0); + + // If the remaining delta is inside the block, update the slot pointer and we are done! + // + if (delta <= pos_in_block) { + this->slot_ -= delta; + break; + } + // Else reduce delta by the number of slots before this one in the current block, plus one + // for the current slot. + // + delta -= (pos_in_block + 1); + + // Move to the last item of the previous block. + // + --this->block_iter_; + this->slot_ = std::prev(this->block_iter_->items_end()); + pos_in_block = this->block_iter_->item_count() - 1; + } + } + } + + isize distance_to(const Self& other) const + { + if (this->block_iter_ == other.block_iter_) { + return std::distance(this->slot_, other.slot_); + } + + // Step forward counting items in each block until we reach the same block. + // + if (this->block_iter_ < other.block_iter_) { + isize delta = this->get_remaining_in_block(); + for (auto iter = std::next(this->block_iter_); iter != other.block_iter_; ++iter) { + delta += iter->item_count(); + } + delta += other.get_item_pos_in_block(); + return delta; + } + // Else step backward. + // + isize delta = this->get_item_pos_in_block(); + for (auto iter = std::prev(this->block_iter_); iter != other.block_iter_; --iter) { + delta += iter->item_count(); + } + delta += other.get_remaining_in_block(); + return -delta; + } + + //+++++++++++-+-+--+----- --- -- - - - - + + isize get_item_pos_in_block() const noexcept + { + return std::distance(this->block_iter_->items_begin(), this->slot_); + } + + isize get_remaining_in_block(isize pos_in_block) const noexcept + { + return this->block_iter_->item_count() - pos_in_block; + } + + isize get_remaining_in_block() const noexcept + { + return this->get_remaining_in_block(this->get_item_pos_in_block()); + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + BlockIterator block_iter_; + const PackedKeyValueSlotPtr* slot_ = nullptr; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp new file mode 100644 index 0000000..d2a9353 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp @@ -0,0 +1,56 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_PACKED_BLOCKED_LEAF_PAGE_SHARDED_LIVE_RANGES_HPP + +#include "packed_blocked_leaf_page.hpp" + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +template FilterModelT> +class PackedBlockedLeafPage::ShardedLiveRanges +{ + public: + using Item = std::pair /*live_item_range*/>; + + //+++++++++++-+-+--+----- --- -- - - - - + + explicit ShardedLiveRanges( + const llfs::PackedArray* block_starts, + BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges) noexcept; + + //+++++++++++-+-+--+----- --- -- - - - - + + Optional peek(); + + Optional next(); + + //+++++++++++-+-+--+----- --- -- - - - - + private: + void advance(); + + usize get_block_count() const noexcept; + + Interval get_block_range(usize block_i) const noexcept; + + void clear_current_range(); + + //+++++++++++-+-+--+----- --- -- - - - - + + const llfs::PackedArray* block_starts_; + usize block_index_; + BasicPiecewiseFilter::LiveSubranges filter_live_ranges_; + Interval current_range_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp new file mode 100644 index 0000000..9192cb2 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp @@ -0,0 +1,170 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_PACKED_BLOCKED_LEAF_PAGE_SHARDED_LIVE_RANGES_IPP + +#include "packed_blocked_leaf_page.sharded_live_ranges.hpp" + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline /*explicit*/ PackedBlockedLeafPage::ShardedLiveRanges::ShardedLiveRanges( + const llfs::PackedArray* block_starts, + BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges) noexcept + : block_starts_{block_starts} + , block_index_{0} + , filter_live_ranges_{std::move(filter_live_ranges)} + , current_range_{0, 0} +{ + this->advance(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline auto PackedBlockedLeafPage::ShardedLiveRanges::peek() -> Optional +{ + if (this->current_range_.empty()) { + return None; + } + return std::make_pair(this->block_index_, this->current_range_); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline auto PackedBlockedLeafPage::ShardedLiveRanges::next() -> Optional +{ + Optional item = this->peek(); + if (item) { + this->advance(); + // std::cerr << ".. " << BATT_INSPECT(this->current_range_) << std::endl; + } + return item; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline void PackedBlockedLeafPage::ShardedLiveRanges::advance() +{ + this->current_range_.lower_bound = this->current_range_.upper_bound; + + Optional> filter_range = this->filter_live_ranges_.peek(); + if (!filter_range) { + return; + } + + // Consume the current range from both current and filter. + // + BATT_CHECK_LE(this->current_range_.upper_bound, filter_range->upper_bound); + filter_range->lower_bound = std::max(filter_range->lower_bound, // + this->current_range_.upper_bound); + + // If the filter range has been consumed, move to the next filter range. + // + if (filter_range->empty()) { + this->filter_live_ranges_.next(); + filter_range = this->filter_live_ranges_.peek(); + + // Once we run out of filter live ranges, we are done. + // + if (!filter_range) { + return; + } + } + + // std::cerr << ".. " << BATT_INSPECT(filter_range) << std::endl; + + const usize block_count = this->get_block_count(); + BATT_CHECK_LT(this->block_index_, block_count); + const usize blocks_remaining = block_count - this->block_index_; + const usize max_probe_steps = BATT_CHECKED_CAST(usize, batt::log2_ceil(blocks_remaining)); + const usize linear_probe_end = this->block_index_ + max_probe_steps; + bool tried_binary_search = false; + + while (this->block_index_ < block_count) { + // Test the intersection of the current block's range with the current filter range; + // if they intersect, then stop here. + // + this->current_range_ = this->get_block_range(this->block_index_) // + .intersection_with(*filter_range); + + if (!this->current_range_.empty()) { + return; + } + + // If we can, continue the linear probe. + // + ++this->block_index_; + if (this->block_index_ <= linear_probe_end) { + continue; + } + + // The binary search fall-back *must* succeed! If we ever find we are about to try it a + // second time, panic. + // + BATT_CHECK(!tried_binary_search); + + // Fall-back to binary search. + // + auto indices = boost::irange(this->block_index_, block_count); + auto iter = + std::lower_bound(indices.begin(), + indices.end(), + *filter_range, + [this](usize i, const Interval& range) { + return Interval::LinearOrder{}(this->get_block_range(i), range); + }); + + // If the first block that might intersect with the filter range is beyond the end of the + // blocks, then we are done. + // + if (iter == indices.end()) { + this->clear_current_range(); + return; + } + + this->block_index_ = *iter; + tried_binary_search = true; + } +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline usize PackedBlockedLeafPage::ShardedLiveRanges::get_block_count() + const noexcept +{ + return this->block_starts_->size() - 1; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline Interval PackedBlockedLeafPage::ShardedLiveRanges::get_block_range( + usize block_i) const noexcept +{ + return Interval{ + (*this->block_starts_)[block_i], + (*this->block_starts_)[block_i + 1], + }; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template FilterModelT> +inline void PackedBlockedLeafPage::ShardedLiveRanges::clear_current_range() +{ + this->current_range_.lower_bound = this->current_range_.upper_bound; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp new file mode 100644 index 0000000..fd9ca6f --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -0,0 +1,366 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#include +// +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include +#include +#include + +namespace { + +using namespace batt::int_types; +using namespace batt::constants; + +using batt::MutableBuffer; +using batt::StableStringStore; +using batt::StatusOr; + +using turtle_kv::EditView; +using turtle_kv::Interval; +using turtle_kv::KeyOrder; +using turtle_kv::KeyView; +using turtle_kv::Optional; +using turtle_kv::pack_blocked_leaf_page; +using turtle_kv::PackedBlockedLeafPage; +using turtle_kv::PackedKeyValueSlotPtr; +using turtle_kv::PiecewiseFilter; +using turtle_kv::random_str; +using turtle_kv::ValueView; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// Plan: +// 1. For different random seeds: +// - generate random set of prefixes (~10% of total keys) +// - generate keys using prefixes, with random values +// - sort +// - pack leaf; verify: +// a. all packed keys present and have right values +// b. any unpacked keys at end missing +// c. randomly generated non-present keys not found +// +TEST(TreePackedBlockedLeafPageTest, Random) +{ + const usize kNumSeeds = 10000; + const usize kLeafPageSize = 1 * kMiB; + const usize kNumPrefixes = 1000; + const usize kMinPrefixSize = 0; + const usize kMaxPrefixSize = 8; + const usize kMinKeySize = 4; + const usize kMaxKeySize = 48; + const usize kMinValueSize = 0; + const usize kMaxValueSize = 200; + const usize kBlockSize = 8192; + + BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); + + std::uniform_int_distribution pick_pct{0, 99}; + std::geometric_distribution pick_prefix_size{0.5}; + std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; + std::geometric_distribution pick_key_size{0.7}; + std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; + + for (usize seed = 803; seed < kNumSeeds; ++seed) { + LOG(INFO) << BATT_INSPECT(seed); + + std::default_random_engine rng{seed}; + + StableStringStore strings; + + //+++++++++++-+-+--+----- --- -- - - - - + // Generate prefixes + // + std::vector prefixes; + { + std::unordered_set used_prefixes; + while (prefixes.size() < kNumPrefixes) { + std::string_view prefix = + random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); + + if (used_prefixes.count(prefix)) { + continue; + } + prefixes.push_back(prefix); + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Generate edits. + // + std::vector edits; + { + usize max_edit_size = 0; + usize max_key_size = 0; + usize total_edits_size = 0; + + std::unordered_set used_keys; + for (;;) { + std::string_view prefix = prefixes[pick_prefix(rng)]; + + std::string_view key = + random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); + + if (used_keys.count(key)) { + continue; + } + used_keys.insert(key); + + std::string_view value = + random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); + + EditView edit{key, ValueView::from_str(value)}; + + const usize edit_size = PackedBlockedLeafPage::packed_edit_size(edit); + + const usize new_max_edit_size = std::max(max_edit_size, edit_size); + const usize new_max_key_size = std::max(max_key_size, key.size()); + + const usize space_available = PackedBlockedLeafPage::estimate_capacity(kLeafPageSize, + kBlockSize, + new_max_key_size, + new_max_edit_size); + + // Stop as soon as adding the next key would exceed the estimated space. + // + if (edit_size + total_edits_size > space_available) { + break; + } + + edits.push_back(edit); + total_edits_size += edit_size; + max_edit_size = new_max_edit_size; + max_key_size = new_max_key_size; + } + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Sort edits by key. + // + std::sort(edits.begin(), edits.end(), KeyOrder{}); + + //+++++++++++-+-+--+----- --- -- - - - - + // Pack a blocked leaf page. + // + using StorageUnit = std::aligned_storage_t<4096, 4096>; + std::vector leaf_storage(kLeafPageSize / sizeof(StorageUnit)); + ASSERT_EQ(sizeof(StorageUnit) * leaf_storage.size(), kLeafPageSize); + + MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; + + StatusOr status_or_packed_leaf = + pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); + + ASSERT_TRUE(status_or_packed_leaf.ok()) << BATT_INSPECT(status_or_packed_leaf.status()); + + const PackedBlockedLeafPage& packed_leaf = PackedBlockedLeafPage::view_of(leaf_buffer); + + ASSERT_EQ(&packed_leaf, *status_or_packed_leaf); + ASSERT_EQ(packed_leaf.min_key(), get_key(edits.front())); + ASSERT_EQ(packed_leaf.max_key(), get_key(edits.back())); + + //+++++++++++-+-+--+----- --- -- - - - - + // Scan over all items in the packed leaf to make sure they are all there. + // + { + PackedBlockedLeafPage::ItemIterator item_iter = packed_leaf.items_begin(); + PackedBlockedLeafPage::ItemIterator items_end = packed_leaf.items_end(); + + std::vector> past_items; + + auto packed_items = packed_leaf.items_seq(); + using Item = decltype(*packed_items.peek()); + Optional prev_key; + Optional prev_item_iter; + + isize position = 0; + + for (const EditView& edit : edits) { + Optional next_packed = packed_items.next(); + + if (prev_key) { + ASSERT_GT(get_key(edit), *prev_key); + } + prev_key = get_key(edit); + + ASSERT_TRUE(next_packed.has_value()); + ASSERT_EQ(get_key(*next_packed), get_key(edit)); + ASSERT_EQ(get_value(*next_packed), get_value(edit)); + + // Test PackedBlockedLeafPage::find_key. + // + const PackedKeyValueSlotPtr* found = packed_leaf.find_key(get_key(edit)); + + ASSERT_NE(found, nullptr); + ASSERT_EQ(found, std::addressof(*item_iter)); + ASSERT_EQ(get_key(*found), get_key(edit)); + ASSERT_EQ(get_value(*found), get_value(edit)) + << BATT_INSPECT_STR(get_key(*found)) << BATT_INSPECT(edit); + + // Test PackedBlockedLeafPage::lower_bound. + // + { + PackedBlockedLeafPage::ItemIterator lb_iter = packed_leaf.lower_bound(get_key(edit)); + + ASSERT_NE(lb_iter, items_end); + ASSERT_EQ(get_key(*lb_iter), get_key(edit)); + ASSERT_EQ(get_value(*lb_iter), get_value(edit)); + } + + ASSERT_EQ(packed_leaf.item_at(position), item_iter); + + ASSERT_NE(item_iter, items_end); + ASSERT_LT(item_iter, items_end); + ASSERT_EQ(std::distance(packed_leaf.items_begin(), item_iter), position); + + if (pick_pct(rng) < 1) { + past_items.push_back(std::make_pair(item_iter, position)); + } + + for (const auto& [past_iter, past_position] : past_items) { + ASSERT_EQ(std::distance(past_iter, item_iter), position - past_position); + ASSERT_EQ(std::distance(item_iter, past_iter), past_position - position); + ASSERT_EQ(past_iter + (position - past_position), item_iter); + ASSERT_EQ(item_iter - (position - past_position), past_iter); + ASSERT_LE(past_iter, item_iter); + ASSERT_GE(item_iter, past_iter) << BATT_INSPECT(position) << BATT_INSPECT(past_position); + } + + if (prev_item_iter) { + ASSERT_EQ(std::next(*prev_item_iter), item_iter); + ASSERT_EQ(*prev_item_iter, std::prev(item_iter)); + } + + prev_item_iter = item_iter; + ++item_iter; + ++position; + } + ASSERT_FALSE(packed_items.peek().has_value()); + } + + //+++++++++++-+-+--+----- --- -- - - - - + // Test ShardedLiveRanges. + // + { + for (usize j = 0; j < 1000; ++j) { + // Drop up to 64 sub-ranges of the leaf. + // + for (usize drop_count = 0; drop_count < 64; ++drop_count) { + std::vector> dropped_ranges; + PiecewiseFilter leaf_filter; + + usize drops_remaining = drop_count; + const u32 item_count = packed_leaf.item_count(); + + u32 next_droppable = 0; + u32 items_dropped = 0; + + for (usize drop_i = 0; drop_i < drop_count; ++drop_i) { + BATT_CHECK_GE(next_droppable, 0); + BATT_CHECK_LT(next_droppable, item_count); + + std::uniform_int_distribution pick_lower_bound{ + next_droppable, + item_count - (drops_remaining * 2 - 1), + }; + const u32 lower_bound_i = pick_lower_bound(rng); + + std::uniform_int_distribution pick_upper_bound{ + lower_bound_i + 1, + item_count - (drops_remaining * 2 - 2), + }; + const u32 upper_bound_i = pick_upper_bound(rng); + + BATT_CHECK_LT(lower_bound_i, upper_bound_i); + BATT_CHECK_GE(lower_bound_i, next_droppable); + + items_dropped += upper_bound_i - lower_bound_i; + + const usize live_count_before = leaf_filter.live().size(); + //----- --- -- - - - - + dropped_ranges.push_back(Interval{lower_bound_i, upper_bound_i}); + leaf_filter.drop_index_range(Interval{lower_bound_i, upper_bound_i}); + //----- --- -- - - - - + const usize live_count_after = leaf_filter.live().size(); + + if (lower_bound_i == 0) { + ASSERT_EQ(live_count_after, live_count_before); + } else { + ASSERT_EQ(live_count_after, live_count_before + 1); + } + + --drops_remaining; + next_droppable = upper_bound_i + 1; + } + + // Verify the number of expected live items. + // + const u32 expected_live_count = item_count - items_dropped; + + if (drop_count > 1) { + ASSERT_GT(expected_live_count, 0) + << BATT_INSPECT_RANGE(dropped_ranges) << BATT_INSPECT(drop_count) + << BATT_INSPECT(item_count); + } + + u32 actual_live_count = 0; + u32 next_possible_live = 0; + u32 next_possible_block = 0; + + packed_leaf.sharded_live_ranges(leaf_filter, Interval{0, item_count}) | + batt::seq::for_each([&](const std::pair>& live_pair) { + const auto [block_index, live_range] = live_pair; + + // std::cerr << BATT_INSPECT(block_index) << BATT_INSPECT(live_range) << std::endl; + + BATT_CHECK_GE(block_index, next_possible_block); + BATT_CHECK_LT(block_index, packed_leaf.block_count()); + BATT_CHECK_GE(live_range.lower_bound, next_possible_live) + << BATT_INSPECT(j) << BATT_INSPECT(drop_count) << BATT_INSPECT(live_range) + << BATT_INSPECT(item_count); + BATT_CHECK_LT(live_range.lower_bound, live_range.upper_bound); + BATT_CHECK_LE(live_range.upper_bound, item_count); + + const Interval block_range = + packed_leaf.item_index_range_of_block(block_index); + + BATT_CHECK_GE(live_range.lower_bound, block_range.lower_bound); + BATT_CHECK_LE(live_range.upper_bound, block_range.upper_bound); + + next_possible_live = live_range.upper_bound; + next_possible_block = block_index; + + actual_live_count += live_range.size(); + }); + + ASSERT_EQ(actual_live_count, expected_live_count) + << BATT_INSPECT(j) << BATT_INSPECT(drop_count); + } + } + } + } +} + +} // namespace diff --git a/src/turtle_kv/tree/packed_leaf_block.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block.hpp similarity index 68% rename from src/turtle_kv/tree/packed_leaf_block.hpp rename to src/turtle_kv/tree/leaf/packed_leaf_block.hpp index 78b55ad..997f0ef 100644 --- a/src/turtle_kv/tree/packed_leaf_block.hpp +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.hpp @@ -9,6 +9,8 @@ #pragma once #define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_HPP +#include "packed_leaf_block_stats.hpp" + #include #include #include @@ -23,11 +25,8 @@ #include #include -#include #include -#include - #include namespace turtle_kv { @@ -41,6 +40,8 @@ struct PackedLeafBlock { class Iterator; + using BlockItemsSeq = batt::SubRangeSeq>; + //+++++++++++-+-+--+----- --- -- - - - - big_u32 magic; // +4 = 4 @@ -106,6 +107,11 @@ struct PackedLeafBlock { return this->items_[this->item_count() - 1]; } + BlockItemsSeq items_seq() const noexcept + { + return batt::as_seq(this->items_slice()); + } + const PackedKeyValueSlotPtr* items_begin() const noexcept { return this->items_; @@ -151,85 +157,6 @@ struct PackedLeafBlock { static_assert(sizeof(PackedLeafBlock) == 8); -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -class PackedLeafBlock::Iterator - : public boost::iterator_facade< // - PackedLeafBlock::Iterator, // <- Derived - const PackedLeafBlock, // <- Value - std::random_access_iterator_tag, // <- CategoryOrTraversal - const PackedLeafBlock&, // <- Reference - isize // <- Difference - > -{ - public: - using Self = Iterator; - using iterator_category = std::random_access_iterator_tag; - using value_type = const PackedLeafBlock; - using reference = const PackedLeafBlock&; - - Iterator() = default; - - explicit Iterator(const PackedLeafBlock* block, isize block_size) noexcept - : block_{block} - , block_size_{block_size} - { - } - - reference dereference() const - { - return *this->block_; - } - - bool equal(const Self& other) const - { - return this->block_ == other.block_ && this->block_size_ == other.block_size_; - } - - void increment() - { - this->advance(1); - } - - void decrement() - { - this->advance(-1); - } - - void advance(isize delta) - { - this->block_ = static_cast( - advance_pointer(this->block_, delta * this->block_size_)); - } - - isize distance_to(const Self& other) const - { - return (byte_distance(this->block_, other.block_)) / this->block_size_; - } - - private: - const PackedLeafBlock* block_ = nullptr; - isize block_size_ = 0; -}; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -struct PackedLeafBlockStats { - usize block_size; - usize item_count; - usize item_slot_bytes; - usize item_ptr_bytes; - - //+++++++++++-+-+--+----- --- -- - - - - - - template - static PackedLeafBlockStats from(const RangeT& src, usize block_size) noexcept; -}; - -BATT_OBJECT_PRINT_IMPL((inline), - PackedLeafBlockStats, - (block_size, item_count, item_slot_bytes, item_ptr_bytes)) - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template @@ -25,48 +26,6 @@ namespace turtle_kv { -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline /*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, - usize dst_size) noexcept -{ - PackedLeafBlockStats stats{ - .block_size = 0, - .item_count = 0, - .item_slot_bytes = 0, - .item_ptr_bytes = 0, - }; - - if (dst_size < sizeof(PackedLeafBlock)) { - return stats; - } - stats.block_size = dst_size; - usize offset = 0; - dst_size -= sizeof(PackedLeafBlock); - offset += sizeof(PackedLeafBlock); - - for (const auto& src_item : src) { - const usize slot_size = packed_key_value_slot_size(src_item); - const usize total_item_size = slot_size + sizeof(PackedKeyValueSlotPtr); - if (dst_size < total_item_size) { - break; - } - stats.item_count += 1; - stats.item_slot_bytes += slot_size; - stats.item_ptr_bytes += sizeof(PackedKeyValueSlotPtr); - dst_size -= total_item_size; - offset += total_item_size; - - if constexpr (false) { - LOG(INFO) << BATT_INSPECT(offset) << BATT_INSPECT_STR(get_key(src_item)) - << BATT_INSPECT(stats.item_count); - } - } - - return stats; -} - //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp new file mode 100644 index 0000000..ade706d --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp @@ -0,0 +1,104 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_ITERATOR_HPP + +#include "packed_leaf_block.hpp" + +#include +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +class PackedLeafBlock::Iterator + : public boost::iterator_facade< // + PackedLeafBlock::Iterator, // <- Derived + const PackedLeafBlock, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + const PackedLeafBlock&, // <- Reference + isize // <- Difference + > +{ + public: + using Self = Iterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = const PackedLeafBlock; + using reference = const PackedLeafBlock&; + + //+++++++++++-+-+--+----- --- -- - - - -- + + Iterator() = default; + + explicit Iterator(const PackedLeafBlock* block, isize block_size) noexcept + : block_{block} + , block_size_{block_size} + { + } + + //+++++++++++-+-+--+----- --- -- - - - -- + + reference dereference() const + { + return *this->block_; + } + + bool equal(const Self& other) const + { + return this->block_ == other.block_ && this->block_size_ == other.block_size_; + } + + void increment() + { + this->advance(1); + } + + void decrement() + { + this->advance(-1); + } + + void advance(isize delta) + { + this->block_ = static_cast( + advance_pointer(this->block_, delta * this->block_size_)); + } + + isize distance_to(const Self& other) const + { + return (byte_distance(this->block_, other.block_)) / this->block_size_; + } + + //+++++++++++-+-+--+----- --- -- - - - -- + + const PackedLeafBlock* block() const noexcept + { + return this->block_; + } + + usize block_size() const noexcept + { + return static_cast(this->block_size_); + } + + isize block_isize() const noexcept + { + return this->block_size_; + } + + //+++++++++++-+-+--+----- --- -- - - - -- + private: + const PackedLeafBlock* block_ = nullptr; + isize block_size_ = 0; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block.test.cpp b/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp similarity index 96% rename from src/turtle_kv/tree/packed_leaf_block.test.cpp rename to src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp index 73a9e1f..f19fefc 100644 --- a/src/turtle_kv/tree/packed_leaf_block.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp @@ -6,14 +6,14 @@ // //+++++++++++-+-+--+----- --- -- - - - - -#include +#include // -#include +#include #include #include -#include "random_str.hpp" +#include #include @@ -119,6 +119,8 @@ TEST(TreePackedLeafBlockTest, Random) } } + // Run empty queries. + // for (usize i = 0; i < kNumNotFoundQueries; ++i) { std::string_view key; for (;;) { @@ -135,6 +137,8 @@ TEST(TreePackedLeafBlockTest, Random) ASSERT_EQ(packed_block.find_key(key), nullptr); } + // Run lower bound queries. + // for (usize i = 0; i < kNumLowerBoundQueries; ++i) { std::string_view key = (i % 2) ? random_str(rng, pick_key_size, diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp new file mode 100644 index 0000000..2d8360f --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp @@ -0,0 +1,38 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_STATS_HPP + +#include + +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +struct PackedLeafBlockStats { + usize block_size; + usize item_count; + usize item_slot_bytes; + usize item_ptr_bytes; + + //+++++++++++-+-+--+----- --- -- - - - - + + template + static PackedLeafBlockStats from(const RangeT& src, usize block_size) noexcept; +}; + +BATT_OBJECT_PRINT_IMPL((inline), + PackedLeafBlockStats, + (block_size, item_count, item_slot_bytes, item_ptr_bytes)) + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp new file mode 100644 index 0000000..a9b9bf7 --- /dev/null +++ b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp @@ -0,0 +1,56 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_STATS_IPP + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +inline /*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, + usize dst_size) noexcept +{ + PackedLeafBlockStats stats{ + .block_size = 0, + .item_count = 0, + .item_slot_bytes = 0, + .item_ptr_bytes = 0, + }; + + if (dst_size < sizeof(PackedLeafBlock)) { + return stats; + } + stats.block_size = dst_size; + usize offset = 0; + dst_size -= sizeof(PackedLeafBlock); + offset += sizeof(PackedLeafBlock); + + for (const auto& src_item : src) { + const usize slot_size = packed_key_value_slot_size(src_item); + const usize total_item_size = slot_size + sizeof(PackedKeyValueSlotPtr); + if (dst_size < total_item_size) { + break; + } + stats.item_count += 1; + stats.item_slot_bytes += slot_size; + stats.item_ptr_bytes += sizeof(PackedKeyValueSlotPtr); + dst_size -= total_item_size; + offset += total_item_size; + + if constexpr (false) { + LOG(INFO) << BATT_INSPECT(offset) << BATT_INSPECT_STR(get_key(src_item)) + << BATT_INSPECT(stats.item_count); + } + } + + return stats; +} + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.cpp deleted file mode 100644 index 25ef37d..0000000 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.cpp +++ /dev/null @@ -1,13 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#include -// - -namespace turtle_kv { -} diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/packed_blocked_leaf_page.hpp deleted file mode 100644 index 5aa3008..0000000 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.hpp +++ /dev/null @@ -1,145 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP - -#include - -#include - -#include - -#include -#include -#include - -#include -#include - -#include -#include - -#include - -namespace turtle_kv { - -struct PackedBlockedLeafPage { - static constexpr u64 kMagic = 0x6456beb7f9558445ull; - - //+++++++++++-+-+--+----- --- -- - - - - - - template - static usize packed_edit_size(const EditT& edit) noexcept - { - return PackedLeafBlock::packed_edit_size(edit); - } - - static usize estimate_capacity(usize leaf_size, - usize block_size, - usize max_key_size, - usize max_edit_size) noexcept; - - //+++++++++++-+-+--+----- --- -- - - - - - - big_u64 magic; // +8 -> 8 - little_u32 item_count; // +4 -> 12 - little_u32 total_packed_size; // +4 -> 16 - little_u32 blocks_per_art_key; // +4 -> 20 - little_u32 block_size_bytes; // +4 -> 24 - little_u32 block_count; // +4 -> 28 - llfs::PackedPointer block0; // +4 -> 32 - - /** \brief Pointer to array that stores, for each block, the starting item index relative to the - * entire leaf. - */ - llfs::PackedPointer> block_starting_item; // +4 -> 36 - - /** \brief Pointer to packed ART index. - */ - llfs::PackedPointer art_block_index; // +4 -> 40 - - u8 pad_[24]; - - //+++++++++++-+-+--+----- --- -- - - - - - - PackedLeafBlock::Iterator blocks_begin() const noexcept - { - return PackedLeafBlock::Iterator{this->block0.get(), (isize)this->block_size_bytes.value()}; - } - - PackedLeafBlock::Iterator blocks_end() const noexcept - { - return this->blocks_begin() + this->block_count; - } - - auto blocks() const noexcept - { - return std::ranges::subrange(this->blocks_begin(), - this->blocks_end()); - } - - auto blocks_seq() const noexcept - { - return batt::as_seq(this->blocks()); - } - - auto items_seq() const noexcept - { - return this->blocks_seq() | batt::seq::map([](const PackedLeafBlock& block) { - return batt::as_seq(block.items_slice()); - }) | - batt::seq::flatten(); - } -}; - -static_assert(sizeof(PackedBlockedLeafPage) == 64); - -template -StatusOr pack_blocked_leaf_page(const usize block_size, - const ItemRangeT& src_items, - const MutableBuffer& dst_buffer) noexcept; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline /*static*/ usize PackedBlockedLeafPage::estimate_capacity(usize leaf_size, - usize block_size, - usize max_key_size, - usize max_edit_size) noexcept -{ - const usize space_after_header = - leaf_size - (sizeof(llfs::PackedPageHeader) + sizeof(PackedBlockedLeafPage)); - - const usize max_block_count = space_after_header / block_size; - - const usize block_starts_size = - sizeof(llfs::PackedArray) + sizeof(little_u32) * max_block_count; - - const usize space_after_block_starts = space_after_header - block_starts_size; - - const usize max_art_size = max_key_size * max_block_count * 2; - - const usize space_after_art = space_after_block_starts - max_art_size; - - BATT_CHECK_EQ(batt::bit_count(block_size), 1) << "Leaf block_size must be a power of 2"; - const usize space_for_blocks = space_after_art & ~(block_size - 1); - const usize block_count = space_for_blocks / block_size; - - const usize max_wasted_per_block = max_edit_size - 1; - const usize min_block_capacity = PackedLeafBlock::capacity(block_size) - max_wasted_per_block; - - const usize final_estimate = block_count * min_block_capacity; - - BATT_CHECK_GT(leaf_size, final_estimate); - - return final_estimate; -} - -} // namespace turtle_kv - -#include "packed_blocked_leaf_page.ipp" diff --git a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp deleted file mode 100644 index ec26d26..0000000 --- a/src/turtle_kv/tree/packed_blocked_leaf_page.test.cpp +++ /dev/null @@ -1,192 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#include -// -#include - -#include -#include - -#include "random_str.hpp" - -#include - -#include -#include - -#include -#include -#include - -namespace { - -using namespace batt::int_types; -using namespace batt::constants; - -using batt::MutableBuffer; -using batt::StableStringStore; -using batt::StatusOr; - -using turtle_kv::EditView; -using turtle_kv::KeyOrder; -using turtle_kv::KeyView; -using turtle_kv::Optional; -using turtle_kv::pack_blocked_leaf_page; -using turtle_kv::PackedBlockedLeafPage; -using turtle_kv::random_str; -using turtle_kv::ValueView; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// Plan: -// 1. For different random seeds: -// - generate random set of prefixes (~10% of total keys) -// - generate keys using prefixes, with random values -// - sort -// - pack leaf; verify: -// a. all packed keys present and have right values -// b. any unpacked keys at end missing -// c. randomly generated non-present keys not found -// -TEST(TreePackedBlockedLeafPageTest, Random) -{ - const usize kNumSeeds = 1000; - const usize kLeafPageSize = 1 * kMiB; - const usize kNumPrefixes = 1000; - const usize kMinPrefixSize = 0; - const usize kMaxPrefixSize = 8; - const usize kMinKeySize = 4; - const usize kMaxKeySize = 48; - const usize kMinValueSize = 0; - const usize kMaxValueSize = 200; - const usize kBlockSize = 8192; - - BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); - - std::geometric_distribution pick_prefix_size{0.5}; - std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; - std::geometric_distribution pick_key_size{0.7}; - std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; - - usize total_keys = 0; - usize total_bytes = 0; - - for (usize seed = 0; seed < kNumSeeds; ++seed) { - std::default_random_engine rng{seed}; - - StableStringStore strings; - - //+++++++++++-+-+--+----- --- -- - - - - - // Generate prefixes - // - std::vector prefixes; - { - std::unordered_set used_prefixes; - while (prefixes.size() < kNumPrefixes) { - std::string_view prefix = - random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); - - if (used_prefixes.count(prefix)) { - continue; - } - prefixes.push_back(prefix); - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Generate edits. - // - std::vector edits; - { - usize max_edit_size = 0; - usize max_key_size = 0; - usize total_edits_size = 0; - - std::unordered_set used_keys; - for (;;) { - std::string_view prefix = prefixes[pick_prefix(rng)]; - - std::string_view key = - random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); - - if (used_keys.count(key)) { - continue; - } - - std::string_view value = - random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); - - EditView edit{key, ValueView::from_str(value)}; - - const usize edit_size = PackedBlockedLeafPage::packed_edit_size(edit); - - const usize new_max_edit_size = std::max(max_edit_size, edit_size); - const usize new_max_key_size = std::max(max_key_size, key.size()); - - const usize space_available = PackedBlockedLeafPage::estimate_capacity(kLeafPageSize, - kBlockSize, - new_max_key_size, - new_max_edit_size); - - // Stop as soon as adding the next key would exceed the estimated space. - // - if (edit_size + total_edits_size > space_available) { - break; - } - - ++total_keys; - total_bytes += edit_size; - - edits.push_back(edit); - total_edits_size += edit_size; - max_edit_size = new_max_edit_size; - max_key_size = new_max_key_size; - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Sort edits by key. - // - std::sort(edits.begin(), edits.end(), KeyOrder{}); - - //+++++++++++-+-+--+----- --- -- - - - - - // Pack a blocked leaf page. - // - using StorageUnit = std::aligned_storage_t<4096, 4096>; - std::vector leaf_storage(kLeafPageSize / sizeof(StorageUnit)); - ASSERT_EQ(sizeof(StorageUnit) * leaf_storage.size(), kLeafPageSize); - - MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; - - StatusOr status_or_packed_leaf = - pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); - - ASSERT_TRUE(status_or_packed_leaf.ok()) << BATT_INSPECT(status_or_packed_leaf.status()); - - const PackedBlockedLeafPage& packed_leaf = **status_or_packed_leaf; - - //+++++++++++-+-+--+----- --- -- - - - - - // - // - { - auto packed_items = packed_leaf.items_seq(); - using Item = decltype(*packed_items.peek()); - for (const EditView& edit : edits) { - Optional next_packed = packed_items.next(); - - ASSERT_TRUE(next_packed.has_value()); - ASSERT_EQ(get_key(*next_packed), get_key(edit)); - ASSERT_EQ(get_value(*next_packed), get_value(edit)); - } - } - } - - std::cerr << BATT_INSPECT(total_keys) << BATT_INSPECT(total_bytes) << std::endl; -} - -} // namespace diff --git a/src/turtle_kv/tree/packed_leaf_block_scanner.hpp b/src/turtle_kv/tree/packed_leaf_block_scanner.hpp new file mode 100644 index 0000000..5f5d81a --- /dev/null +++ b/src/turtle_kv/tree/packed_leaf_block_scanner.hpp @@ -0,0 +1,75 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_SCANNER_HPP + +#include + +#include + +#include + +#include + +#include + +namespace turtle_kv { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template +concept PackedLeafBlockProvider = requires(T& provider, usize block_index) { + { provider.get_block(block_index) } -> std::convertible_to>; +}; + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +template +class PackedLeafBlockScanner +{ + public: + class Impl + { + public: + using Item = StatusOr; + + Optional poll() noexcept + { + } + + Optional next() noexcept + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + void advance() noexcept + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + PackedBlockedLeafPage::HeaderShardView header_; + + BlockProviderT& provider_; + + PiecewiseFilter& filter_; + + usize block_index_; + + Optional> block_; + }; + + //+++++++++++-+-+--+----- --- -- - - - - + + private: + Impl* impl_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_node_page.cpp b/src/turtle_kv/tree/packed_node_page.cpp index 6dbdd78..904b83e 100644 --- a/src/turtle_kv/tree/packed_node_page.cpp +++ b/src/turtle_kv/tree/packed_node_page.cpp @@ -1,3 +1,11 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + #include // @@ -8,6 +16,7 @@ #include #include +#include #include diff --git a/src/turtle_kv/tree/testing/fake_segment.hpp b/src/turtle_kv/tree/testing/fake_segment.hpp index 8297a28..860f722 100644 --- a/src/turtle_kv/tree/testing/fake_segment.hpp +++ b/src/turtle_kv/tree/testing/fake_segment.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -102,11 +103,11 @@ struct FakeSegment { { const bool inactive = this->active_pivots_.is_empty(); if (inactive) { - Slice> live_ranges = this->filter_.live(); - BATT_CHECK_EQ(live_ranges.size(), 1) << BATT_INSPECT(live_ranges); - BATT_CHECK_EQ(live_ranges[0].upper_bound, PiecewiseFilter::kMaxUpperBound) - << BATT_INSPECT(live_ranges); - } + Slice> live_ranges = this->filter_.live(); + BATT_CHECK_EQ(live_ranges.size(), 1) << BATT_INSPECT(live_ranges); + BATT_CHECK_EQ(live_ranges[0].upper_bound, PiecewiseFilter::kMaxUpperBound) + << BATT_INSPECT(live_ranges); + } return inactive; } diff --git a/src/turtle_kv/util/packed_piecewise_filter_view.hpp b/src/turtle_kv/util/packed_piecewise_filter_view.hpp new file mode 100644 index 0000000..069cc11 --- /dev/null +++ b/src/turtle_kv/util/packed_piecewise_filter_view.hpp @@ -0,0 +1,305 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PACKED_PIECEWISE_FILTER_VIEW_HPP + +#include "piecewise_filter_storage_model.concept.hpp" + +#include +#include +#include + +#include + +#include + +#include + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Read-only model of PiecewiseFilterStorageModel for packed filters. + * + * Packed piecewise filters are represented as an array of integers, which are the boundaries + * between live and dropped intervals, plus an additional boolean/bit denoting whether the interval + * from the global minimum to the first stored boundary is live or dropped (`start_is_live`). + * + * The global minimum and maximum bounds are never stored in the packed representation. Instead, + * the minimum is implied via the `start_is_live` bit, and the maximum by whether the number of + * stored bounds (plus the implicit first bound, if start_is_live == true) is even or odd. If it is + * odd, then it is implied that there is a final bound equal to the global maximum. + * + * Examples: + * + * Live Intervals: {[0, 10), [20, 30), [40, 50)} + * Packed: start_is_live=1, {10, 20, 30, 40, 50} + * + * Live Intervals: {[0, 10), [20, 30), [40, +inf)} + * Packed: start_is_live=1, {10, 20, 30, 40} + * + * Live Intervals: {[10, 20), [30, 40), [50, 60)} + * Packed: start_is_live=0, {10, 20, 30, 40, 50, 60} + * + * Live Intervals: {[10, 20), [30, 40), [50, +inf)} + * Packed: start_is_live=0, {10, 20, 30, 40, 50} + */ +class PackedPiecewiseFilterView +{ + public: + //----- --- -- - - - - + + // Forward-declaration; the type returned by this->begin(), this->end() + // + class const_iterator; + + /** \brief The boundary integer type. Must be unsigned. + */ + using OffsetT = const little_u32; + + /** \brief The live range type; what `iterator` iterates over. + */ + using value_type = Interval; + + /** \brief Non-const iterator aliases const_iterator, since this storage model is read-only. + */ + using iterator = const_iterator; + + // Forward-declaration; defined below. + // + friend const Slice& as_const_slice(const PackedPiecewiseFilterView& view); + + //----- --- -- - - - - + + /** \brief Constructs an PackedPiecewiseFilterView representing the live interval [0, +inf). + */ + PackedPiecewiseFilterView() = default; + + /** \brief Destructs the PackedPiecewiseFilterView. + */ + ~PackedPiecewiseFilterView() = default; + + /** \brief PackedPiecewiseFilterView is copy constructible. + */ + PackedPiecewiseFilterView(const PackedPiecewiseFilterView&) = default; + + /** \brief PackedPiecewiseFilterView is copy assignable. + */ + PackedPiecewiseFilterView& operator=(const PackedPiecewiseFilterView&) = default; + + /** \brief Constructs PackedPiecewiseFilterView from the packed data in the arguments. + * + * See the class-level description for details on what `values` and `start_is_live` represent. + */ + explicit PackedPiecewiseFilterView(const Slice& values, + bool start_is_live) noexcept + : values_{values} + , implicit_first_{start_is_live ? 1 : 0} + , size_{(this->implicit_first_ + this->values_.size() + 1) & ~i32{1}} + { + } + + //----- --- -- - - - - + + /** \brief Returns an iterator to the first live interval in this filter. + */ + const_iterator begin() const noexcept; + + /** \brief Returns an iterator to one past the last live interval in this filter. + */ + const_iterator end() const noexcept; + + /** \brief Returns the number of live intervals in the filter; same as + * `std::distance(this->begin(), this->end())`. + */ + usize size() const noexcept; + + /** \brief Returns true iff `this->size() == 0`. + */ + bool empty() const noexcept; + + /** \brief Returns the `i`-th live interval in the filter. Behavior is undefined if `i` is not + * less than `this->size()`. + */ + Interval operator[](isize i) const noexcept; + + //----- --- -- - - - - + private: + /** \brief Points at the stored boundaries, as described in the class-level doc. + */ + Slice values_; + + /** \brief Set to 1 if `start_is_live`, else 0. + */ + i32 implicit_first_ = 1; + + /** \brief The number of live intervals in the filter. + */ + i32 size_ = 1; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Returns a const reference to the stored values referenced by `view`. + */ +inline const Slice& as_const_slice(const PackedPiecewiseFilterView& view) +{ + return view.values_; +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief Read-only, random access iterator over the live intervals of a packed piecewise filter. + */ +class PackedPiecewiseFilterView::const_iterator + : public boost::iterator_facade< // + PackedPiecewiseFilterView::const_iterator, // <- Derived + Interval, // <- Value + std::random_access_iterator_tag, // <- CategoryOrTraversal + Interval, // <- Reference + isize // <- Difference + > +{ + public: + using Self = const_iterator; + using iterator_category = std::random_access_iterator_tag; + using value_type = Interval; + using reference = Interval; + + //+++++++++++-+-+--+----- --- -- - - - - + + /** \brief Constructs an invalid iterator. + */ + const_iterator() noexcept : view_{nullptr}, pos_{0} + { + } + + /** \brief Constructs an iterator to the `pos`-th live interval of `view`. + * + * `view` must remain in-scope while this object exists. + */ + const_iterator(const PackedPiecewiseFilterView* view, isize pos) noexcept : view_{view}, pos_{pos} + { + } + + //+++++++++++-+-+--+----- --- -- - - - - + + /** \brief Returns the live interval at the current position. + */ + reference dereference() const + { + return (*this->view_)[this->pos_]; + } + + /** \brief Returns true iff this iterator points to the same live interval of the same filter as + * `other`. + */ + bool equal(const Self& other) const + { + return this->view_ == other.view_ && this->pos_ == other.pos_; + } + + /** \brief Moves this iterator forward by one. + */ + void increment() + { + ++this->pos_; + } + + /** \brief Moves this iterator backward by one. + */ + void decrement() + { + --this->pos_; + } + + /** \brief Moves this iterator by `delta`. + */ + void advance(isize delta) + { + this->pos_ += delta; + } + + /** \brief Returns the number of steps required to advance this iterator so it is equivalent to + * `other`. Will panic if this and other do not point at the same filter view. + */ + isize distance_to(const Self& other) const + { + BATT_CHECK_EQ(this->view_, other.view_); + return other.pos_ - this->pos_; + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + /** \brief Pointer to the filter view over which we are iterating. + */ + const PackedPiecewiseFilterView* view_; + + /** \brief The (logical) position of this iterator within `view_`. + */ + isize pos_; +}; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline auto PackedPiecewiseFilterView::begin() const noexcept -> const_iterator +{ + return const_iterator{this, 0}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline auto PackedPiecewiseFilterView::end() const noexcept -> const_iterator +{ + return const_iterator{this, static_cast(this->size())}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline usize PackedPiecewiseFilterView::size() const noexcept +{ + return this->size_; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline bool PackedPiecewiseFilterView::empty() const noexcept +{ + return this->size_ == 0; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +inline Interval PackedPiecewiseFilterView::operator[](isize i) const noexcept +{ + // Cached for brevity below. + // + const isize n = this->values_.size(); + + // The index within this->values_ of the i-th live interval's lower bound. + // May be negative if the first boundary is implicit (0) + // + const isize j0 = i * 2 - this->implicit_first_; + + // The index within this->values_ of the i-th live interval's upper bound. + // May be past the end of this->values_ if the last boundary is implicit (+inf) + // + const isize j1 = j0 + 1; + + const u32 lower_bound = (j0 < 0) ? std::numeric_limits::min() : this->values_[j0].value(); + const u32 upper_bound = (j1 < n) ? this->values_[j1].value() : std::numeric_limits::max(); + + return Interval{lower_bound, upper_bound}; +} + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- + +static_assert(PiecewiseFilterStorageModel); + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.hpp b/src/turtle_kv/util/piecewise_filter.hpp index f524319..143dc43 100644 --- a/src/turtle_kv/util/piecewise_filter.hpp +++ b/src/turtle_kv/util/piecewise_filter.hpp @@ -9,6 +9,8 @@ #pragma once #define TURTLE_KV_UTIL_PIECEWISE_FILTER_HPP +#include "piecewise_filter_storage_model.concept.hpp" + #include #include #include @@ -26,11 +28,15 @@ namespace turtle_kv { /** \brief A representation of a filtered range of items. */ -template -class PiecewiseFilter +template ModelT> +class BasicPiecewiseFilter : private ModelT { public: - using Self = PiecewiseFilter; + using Self = BasicPiecewiseFilter; + + using ConstIterator = typename ModelT::const_iterator; + + class LiveSubranges; static_assert(std::is_integral::value && std::is_unsigned::value, "Offset must be an unsigned integer type!"); @@ -46,14 +52,15 @@ class PiecewiseFilter /** \brief Creates and returns a PiecewiseFilter instance from a range of intervals that contain * the live item indexes. */ - static StatusOr from_live(const Slice>& live); + static StatusOr from_live(const Slice>& live) + requires PiecewiseFilterMutableStorageModel; //+++++++++++-+-+--+----- --- -- - - - - /** \brief Constructs a default instance of a PiecewiseFilter object, initialized with no item * range and filtered items. */ - PiecewiseFilter() noexcept; + BasicPiecewiseFilter() noexcept; //+++++++++++-+-+--+----- --- -- - - - - @@ -63,7 +70,8 @@ class PiecewiseFilter * * \return The new dropped interval that coincides with `i`. */ - Interval drop_index_range(Interval i); + Interval drop_index_range(Interval i) + requires PiecewiseFilterMutableStorageModel; /** \brief Returns whether or not the item at index `i` has been filtered out. * @@ -99,7 +107,13 @@ class PiecewiseFilter /** \brief Merges two filters in place, taking the union of the live intervals. */ - void merge(const PiecewiseFilter& other); + void merge(const Self& other) + requires PiecewiseFilterMutableStorageModel; + + /** \brief Returns a seq of Interval that is the intersection of `i` and the live ranges + * of this filter. + */ + LiveSubranges live_subranges_of(Interval i) const; /** \brief Validate the state of the live intervals. */ @@ -109,11 +123,22 @@ class PiecewiseFilter //+++++++++++-+-+--+----- --- -- - - - - private: - /** \brief The range of filtered out item indexes. - */ - SmallVec, 64> live_; + BATT_ALWAYS_INLINE ModelT& live_() noexcept + { + return *this; + } + + BATT_ALWAYS_INLINE const ModelT& live_() const noexcept + { + return *this; + } }; +template +using PiecewiseFilter = BasicPiecewiseFilter, 64>>; + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// template inline Interval drop_item_range(PiecewiseFilter& filter, const Slice& items, @@ -127,6 +152,5 @@ inline Interval drop_item_range(PiecewiseFilter& filter, return filter.drop_index_range(Interval{start_i, end_i}); } -} // namespace turtle_kv -#include +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.ipp b/src/turtle_kv/util/piecewise_filter.ipp index 3456043..b892275 100644 --- a/src/turtle_kv/util/piecewise_filter.ipp +++ b/src/turtle_kv/util/piecewise_filter.ipp @@ -11,20 +11,19 @@ #include "piecewise_filter.hpp" -#include - namespace turtle_kv { //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -/*static*/ StatusOr> PiecewiseFilter::from_live( - const Slice>& live) +template ModelT> +/*static*/ StatusOr> +BasicPiecewiseFilter::from_live(const Slice>& live) + requires PiecewiseFilterMutableStorageModel { - PiecewiseFilter filter; - filter.live_.clear(); + Self filter; - filter.live_.insert(filter.live_.end(), live.begin(), live.end()); + filter.live_().clear(); + filter.live_().insert(filter.live_().end(), live.begin(), live.end()); if (!filter.check_invariants()) { return Status{::batt::StatusCode::kInvalidArgument}; @@ -35,16 +34,16 @@ template //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -PiecewiseFilter::PiecewiseFilter() noexcept - : live_{{Interval{Self::kMinLowerBound, Self::kMaxUpperBound}}} +template ModelT> +BasicPiecewiseFilter::BasicPiecewiseFilter() noexcept + : ModelT{{Interval{Self::kMinLowerBound, Self::kMaxUpperBound}}} { } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -bool PiecewiseFilter::check_invariants() const +template ModelT> +bool BasicPiecewiseFilter::check_invariants() const { Optional prev_upper_bound = None; @@ -52,7 +51,7 @@ bool PiecewiseFilter::check_invariants() const // - all intervals are in non-decreasing order // - no intervals overlap or are adjacent (i.e., prev.upper_bound == next.lower_bound) // - for (const Interval& range : this->live_) { + for (const Interval& range : this->live_()) { // If a range has the minimum lower bound, it must be the first. // if (range.lower_bound == Self::kMinLowerBound && prev_upper_bound) { @@ -80,15 +79,16 @@ bool PiecewiseFilter::check_invariants() const //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -Interval PiecewiseFilter::drop_index_range(Interval to_drop) +template ModelT> +Interval BasicPiecewiseFilter::drop_index_range(Interval to_drop) + requires PiecewiseFilterMutableStorageModel { if (to_drop.empty()) { return to_drop; } - auto [first, last] = std::equal_range(this->live_.begin(), - this->live_.end(), + auto [first, last] = std::equal_range(this->live_().begin(), + this->live_().end(), to_drop, typename Interval::LinearOrder{}); @@ -98,7 +98,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t // the return value bounds correctly. // if (first == last || to_drop.lower_bound < first->lower_bound) { - if (first != this->live_.begin()) { + if (first != this->live_().begin()) { // We are starting in a live interval gap (dropped region), so we extend to the previous live // interval's upper bound. // @@ -109,7 +109,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t } if (first == last || to_drop.upper_bound >= std::prev(last)->upper_bound) { - if (last != this->live_.end()) { + if (last != this->live_().end()) { // We are ending in a live interval gap, so we extend to the next live interval's start. // dropped.upper_bound = last->lower_bound; @@ -164,7 +164,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t // Process all overlapping intervals with `to_drop`. // - while (first != this->live_.end()) { + while (first != this->live_().end()) { if (first->lower_bound >= to_drop.upper_bound) { // Interval is entirely after `to_drop`, so there is nothing left to process. // @@ -177,7 +177,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t // Interval right_half{to_drop.upper_bound, first->upper_bound}; first->upper_bound = to_drop.lower_bound; - this->live_.insert(std::next(first), right_half); + this->live_().insert(std::next(first), right_half); return dropped; } else { // Cases 2b and 4. @@ -193,7 +193,7 @@ Interval PiecewiseFilter::drop_index_range(Interval t } else { // Case 1. // - first = this->live_.erase(first); + first = this->live_().erase(first); } } @@ -202,41 +202,41 @@ Interval PiecewiseFilter::drop_index_range(Interval t //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -Slice> PiecewiseFilter::live() const +template ModelT> +Slice> BasicPiecewiseFilter::live() const { - return as_const_slice(this->live_); + return as_const_slice(this->live_()); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -bool PiecewiseFilter::live_at_index(OffsetT i) const +template ModelT> +bool BasicPiecewiseFilter::live_at_index(OffsetT i) const { return this->live_lower_bound(i) == i; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -OffsetT PiecewiseFilter::live_lower_bound(OffsetT i) const +template ModelT> +OffsetT BasicPiecewiseFilter::live_lower_bound(OffsetT i) const { // Compute the live interval which could contain `i`. // - auto iter = std::lower_bound(this->live_.begin(), - this->live_.end(), + auto iter = std::lower_bound(this->live_().begin(), + this->live_().end(), i, typename Interval::LinearOrder{}); // Check if current interval contains `i`. // - if (iter != this->live_.end() && iter->contains(i)) { + if (iter != this->live_().end() && iter->contains(i)) { return i; } // `i` is in a dropped range, so we return the start of the next live interval. // - if (iter != this->live_.end()) { + if (iter != this->live_().end()) { return iter->lower_bound; } @@ -247,22 +247,22 @@ OffsetT PiecewiseFilter::live_lower_bound(OffsetT i) const //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -Interval PiecewiseFilter::find_live_range(Interval i) const +template ModelT> +Interval BasicPiecewiseFilter::find_live_range(Interval i) const { OffsetT start_i = i.lower_bound; OffsetT end_i = i.upper_bound; BATT_CHECK_LE(start_i, end_i); - auto iter = std::lower_bound(this->live_.begin(), - this->live_.end(), + auto iter = std::lower_bound(this->live_().begin(), + this->live_().end(), start_i, typename Interval::LinearOrder{}); // Check if current interval contains or starts at `start_i`. // - if (iter != this->live_.end()) { + if (iter != this->live_().end()) { if (iter->contains(start_i)) { OffsetT live_end = std::min(end_i, iter->upper_bound); return Interval{start_i, live_end}; @@ -283,25 +283,26 @@ Interval PiecewiseFilter::find_live_range(Interval i) //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -void PiecewiseFilter::merge(const PiecewiseFilter& other) +template ModelT> +void BasicPiecewiseFilter::merge(const Self& other) + requires PiecewiseFilterMutableStorageModel { // If other has no live intervals, we are done. // - if (other.live_.empty()) { + if (other.live_().empty()) { return; } // If this has no live intervals, copy from other. // - if (this->live_.empty()) { - this->live_.insert(this->live_.end(), other.live_.begin(), other.live_.end()); + if (this->live_().empty()) { + this->live_().insert(this->live_().end(), other.live_().begin(), other.live_().end()); BATT_CHECK(this->check_invariants()); return; } SmallVec, 64> merged_intervals; - merged_intervals.reserve(this->live_.size() + other.live_.size()); + merged_intervals.reserve(this->live_().size() + other.live_().size()); usize i = 0; usize j = 0; @@ -329,42 +330,57 @@ void PiecewiseFilter::merge(const PiecewiseFilter& other) // Merge the live intervals arrays. // - while (i < this->live_.size() && j < other.live_.size()) { - if (this->live_[i].lower_bound <= other.live_[j].lower_bound) { - add_interval(this->live_[i]); + while (i < this->live_().size() && j < other.live_().size()) { + if (this->live_()[i].lower_bound <= other.live_()[j].lower_bound) { + add_interval(this->live_()[i]); ++i; } else { - add_interval(other.live_[j]); + add_interval(other.live_()[j]); ++j; } } - // Add remaining intervals from this->live_. + // Add remaining intervals.. // - while (i < this->live_.size()) { - add_interval(this->live_[i]); + while (i < this->live_().size()) { + add_interval(this->live_()[i]); ++i; } // Add remaining intervals from other.live_. // - while (j < other.live_.size()) { - add_interval(other.live_[j]); + while (j < other.live_().size()) { + add_interval(other.live_()[j]); ++j; } - this->live_ = std::move(merged_intervals); + this->live_() = std::move(merged_intervals); BATT_CHECK(this->check_invariants()); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -template -SmallFn PiecewiseFilter::dump() const +template ModelT> +SmallFn BasicPiecewiseFilter::dump() const { return [this](std::ostream& out) { - out << batt::dump_range(this->live_); + out << batt::dump_range(this->live_()); }; } + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +auto BasicPiecewiseFilter::live_subranges_of(Interval query_range) const + -> LiveSubranges +{ + const auto [first, last] = std::equal_range(this->live_().begin(), + this->live_().end(), + query_range, + typename Interval::LinearOrder{}); + + return LiveSubranges{query_range, std::ranges::subrange(first, last)}; +} + } // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.live_subranges.hpp b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp new file mode 100644 index 0000000..c2bf50c --- /dev/null +++ b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp @@ -0,0 +1,88 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PIECEWISE_FILTER_LIVE_SUBRANGES_HPP + +#include "piecewise_filter.hpp" + +namespace turtle_kv { + +//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- +// +/** \brief A (batt) Seq over the sub-ranges of a PiecewiseFilter which match some interval (the + * `query_range`). + */ +template ModelT> +class BasicPiecewiseFilter::LiveSubranges +{ + public: + using Iterator = PiecewiseFilter::ConstIterator; + using Item = Interval; + + //+++++++++++-+-+--+----- --- -- - - - - + + /** \brief Constructs a LiveSubranges seq containing the passed range of live intervals (`match`), + * with the first and last element clamped to the `query_range`. + * + * `match` *must* not extend more than one live interval past `query_range` at the front or back. + */ + explicit LiveSubranges(Interval query_range, + std::ranges::subrange match) noexcept + : clamp_lower_{query_range.lower_bound} + , clamp_upper_{query_range.upper_bound} + , match_{match} + { + } + + /** \brief Returns the current live subrange, or None if the seq has been fully consumed. + */ + Optional peek() + { + if (this->match_.empty()) { + return None; + } + Interval item = this->match_.front(); + if (this->clamp_lower_) { + item.lower_bound = std::max(item.lower_bound, *this->clamp_lower_); + } + if (this->match_.size() == 1 && this->clamp_upper_) { + item.upper_bound = std::min(item.upper_bound, *this->clamp_upper_); + } + return item; + } + + /** \brief Returns the current live subrange, or None if the seq has been fully consumed, + * consuming the returned item. + */ + Optional next() + { + Optional item = this->peek(); + if (item) { + this->clamp_lower_ = None; + this->match_.advance(1); + } + return item; + } + + //+++++++++++-+-+--+----- --- -- - - - - + private: + /** \brief The lower bound to which to clamp the first interval of `this->match_`. + */ + Optional clamp_lower_; + + /** \brief The upper bound to which to clamp the last interval of `this->match_`. + */ + Optional clamp_upper_; + + /** \brief The current subrange of the live intervals in the filter. + */ + std::ranges::subrange match_; +}; + +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.test.cpp b/src/turtle_kv/util/piecewise_filter.test.cpp index d4c6ee4..fe4d3de 100644 --- a/src/turtle_kv/util/piecewise_filter.test.cpp +++ b/src/turtle_kv/util/piecewise_filter.test.cpp @@ -13,8 +13,15 @@ #include #include +#include +#include + #include +#include +#include +#include + #include #include #include @@ -39,6 +46,7 @@ using turtle_kv::drop_item_range; using llfs::KeyRangeOrder; +using batt::mask_from_interval; using batt::StableStringStore; //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -138,7 +146,7 @@ TEST(PiecewiseFilterTest, QueryTest) auto iter = live_items.lower_bound(start_i); Interval expected_range; - + if (iter == live_items.end() || *iter >= end_i) { expected_range = Interval{end_i, end_i}; } else { @@ -284,4 +292,67 @@ TEST(PiecewiseFilterTest, KeyQueryTest) EXPECT_TRUE(filter.check_invariants()); } } -} // namespace \ No newline at end of file + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST(PiecewiseFilterTest, LiveSubranges) +{ + std::uniform_int_distribution pick_bound{0, 64}; + + const auto pick_interval = [&](auto& rng) { + Interval i{pick_bound(rng), pick_bound(rng)}; + if (i.upper_bound < i.lower_bound) { + std::swap(i.lower_bound, i.upper_bound); + } + return i; + }; + + std::array, 1> init_live{{{0, 64}}}; + + const usize n_seeds = 10000000; + const usize n_drops = 10; + const usize n_queries = 15; + const usize first_seed = 0; + + for (usize seed_i = first_seed; seed_i < first_seed + n_seeds; ++seed_i) { + std::default_random_engine rng{seed_i}; + + PiecewiseFilter filter = + BATT_OK_RESULT_OR_PANIC(PiecewiseFilter::from_live(batt::as_slice(init_live))); + + const auto query_as_bits = [&](Interval query) { + u64 bits = 0; + filter.live_subranges_of(query) | batt::seq::for_each([&bits](const Interval& live) { + bits |= mask_from_interval(live); + }); + return bits; + }; + + u64 filter_state = ~u64{0}; + + for (usize i = 0; i < n_drops; ++i) { + const Interval drop_interval = pick_interval(rng); + const u64 drop_mask = mask_from_interval(drop_interval); + filter_state &= ~drop_mask; + filter.drop_index_range(drop_interval); + + if constexpr (false) { + std::cerr << BATT_INSPECT(std::bitset<64>{filter_state}) << BATT_INSPECT(filter.dump()) + << std::endl; + } + + for (usize j = 0; j < n_queries; ++j) { + const Interval query_interval = pick_interval(rng); + const u64 query_mask = mask_from_interval(query_interval); + const u64 expected_bits = query_mask & filter_state; + const u64 actual_bits = query_as_bits(query_interval); + + ASSERT_EQ(std::bitset<64>{expected_bits}, std::bitset<64>{actual_bits}) + << BATT_INSPECT(seed_i) << BATT_INSPECT(query_interval) + << BATT_INSPECT(query_interval.size()) << BATT_INSPECT(std::bitset<64>{query_mask}); + } + } + } +} + +} // namespace diff --git a/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp new file mode 100644 index 0000000..6dd8701 --- /dev/null +++ b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp @@ -0,0 +1,60 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PIECEWISE_FILTER_STORAGE_MODEL_CONCEPT_HPP + +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace turtle_kv { + +template +concept PiecewiseFilterStorageModel = requires(const T& model, + T& src, + T& dst, + Interval interval, + usize i, + std::ostream& out) { + typename T::value_type; + typename T::iterator; + typename T::const_iterator; + + { model.begin() }; + { model.end() }; + { std::begin(model) } -> std::same_as; + { std::end(model) } -> std::same_as; + { as_const_slice(model) }; + { model.empty() } -> std::convertible_to; + { model.size() } -> std::convertible_to; + { *model.begin() } -> std::convertible_to&>; + { model[i] } -> std::convertible_to&>; + { dst = std::move(src) }; + { out << batt::dump_range(model) } -> std::same_as; +}; + +template +concept PiecewiseFilterMutableStorageModel = + PiecewiseFilterStorageModel && + requires(T model, T other, Interval interval, usize i, std::ostream& out) { + { model.clear() }; + { model.erase(model.end()) }; + { model.insert(model.end(), interval) }; + { model.insert(model.end(), model.begin(), model.end()) }; + { model[i] = interval }; + }; + +} // namespace turtle_kv From 4cec2199a4394db49dfe468e14d62eae54a43501 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Wed, 24 Jun 2026 15:47:42 -0400 Subject: [PATCH 07/13] Update requirements. --- conan.lock | 9 +++------ conanfile.py | 6 +++--- .../tree/leaf/packed_blocked_leaf_page.test.cpp | 4 ++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/conan.lock b/conan.lock index b1f39f1..1cb73b8 100644 --- a/conan.lock +++ b/conan.lock @@ -2,8 +2,8 @@ "version": "0.5", "requires": [ "abseil/20250127.0", - "artc/0.0.1.dev", - "batteries/0.71.1", + "artc/0.1.0", + "batteries/0.72.0", "boost/1.88.0", "bzip2/1.0.8", "cli11/2.5.0", @@ -16,7 +16,7 @@ "libpfm4/4.13.0", "libunwind/1.8.1", "liburing/2.11", - "llfs/0.47.1", + "llfs/0.47.2", "openssl/3.6.0", "pcg-cpp/cci.20220409", "protobuf/3.21.12", @@ -50,9 +50,6 @@ ], "boost/[>=1.84.0 <2]": [ "boost/1.88.0" - ], - "batteries/[>=0.60.2 <2]": [ - "batteries/0.70.4.dev2" ] }, "config_requires": [] diff --git a/conanfile.py b/conanfile.py index 8f6b9f6..3474dd2 100644 --- a/conanfile.py +++ b/conanfile.py @@ -89,11 +89,11 @@ def requirements(self): } self.requires("abseil/[>=20260107.1]", **VISIBLE, **OVERRIDE) - self.requires("artc/[>=0.0.1 <1]") - self.requires("batteries/[>=0.71.1 <1]", **VISIBLE, **OVERRIDE) + self.requires("artc/[>=0.1.0 <1]") + self.requires("batteries/[>=0.72.0 <1]", **VISIBLE, **OVERRIDE) self.requires("boost/[>=1.88.0 <2]", **VISIBLE, **OVERRIDE) self.requires("glog/[>=0.7.1 <1]", **VISIBLE) - self.requires("llfs/[>=0.47.0 <1]", **VISIBLE) + self.requires("llfs/[>=0.47.2 <1]", **VISIBLE) self.requires("pcg-cpp/[>=cci.20220409]", **VISIBLE) self.requires("yaml-cpp/[>=0.9.0 <1]") self.requires("zlib/1.3.1", **OVERRIDE) diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index fd9ca6f..3b4add5 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -64,7 +64,7 @@ using turtle_kv::ValueView; // TEST(TreePackedBlockedLeafPageTest, Random) { - const usize kNumSeeds = 10000; + const usize kNumSeeds = 100; const usize kLeafPageSize = 1 * kMiB; const usize kNumPrefixes = 1000; const usize kMinPrefixSize = 0; @@ -84,7 +84,7 @@ TEST(TreePackedBlockedLeafPageTest, Random) std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; for (usize seed = 803; seed < kNumSeeds; ++seed) { - LOG(INFO) << BATT_INSPECT(seed); + LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); std::default_random_engine rng{seed}; From 5240d180ea61ce962b8a612ea87137e1bc205a9d Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 28 Jun 2026 11:11:08 -0400 Subject: [PATCH 08/13] Refactor PiecewiseFilter test utils. --- src/turtle_kv/import/interval.hpp | 3 - .../leaf/packed_blocked_leaf_page.test.cpp | 60 ++------ src/turtle_kv/util/piecewise_filter.test.cpp | 43 +++--- src/turtle_kv/util/piecewise_filter.test.hpp | 129 ++++++++++++++++++ ...piecewise_filter_storage_model.concept.hpp | 1 - 5 files changed, 165 insertions(+), 71 deletions(-) create mode 100644 src/turtle_kv/util/piecewise_filter.test.hpp diff --git a/src/turtle_kv/import/interval.hpp b/src/turtle_kv/import/interval.hpp index 53167ea..33cfe08 100644 --- a/src/turtle_kv/import/interval.hpp +++ b/src/turtle_kv/import/interval.hpp @@ -2,9 +2,6 @@ #include -#include -#include - namespace turtle_kv { using batt::BasicInterval; diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index 3b4add5..03c926e 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include @@ -64,7 +65,9 @@ using turtle_kv::ValueView; // TEST(TreePackedBlockedLeafPageTest, Random) { - const usize kNumSeeds = 100; + const usize kFirstSeed = 0; + const usize kNumSeeds = 1000; + const usize kLastSeed = kFirstSeed + kNumSeeds; const usize kLeafPageSize = 1 * kMiB; const usize kNumPrefixes = 1000; const usize kMinPrefixSize = 0; @@ -83,7 +86,7 @@ TEST(TreePackedBlockedLeafPageTest, Random) std::geometric_distribution pick_key_size{0.7}; std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; - for (usize seed = 803; seed < kNumSeeds; ++seed) { + for (usize seed = kFirstSeed; seed < kLastSeed; ++seed) { LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); std::default_random_engine rng{seed}; @@ -264,56 +267,21 @@ TEST(TreePackedBlockedLeafPageTest, Random) // Test ShardedLiveRanges. // { - for (usize j = 0; j < 1000; ++j) { + for (usize j = 0; j < 10000; ++j) { // Drop up to 64 sub-ranges of the leaf. // for (usize drop_count = 0; drop_count < 64; ++drop_count) { - std::vector> dropped_ranges; PiecewiseFilter leaf_filter; - usize drops_remaining = drop_count; - const u32 item_count = packed_leaf.item_count(); - - u32 next_droppable = 0; + std::vector> dropped_ranges; u32 items_dropped = 0; + const u32 item_count = packed_leaf.item_count(); - for (usize drop_i = 0; drop_i < drop_count; ++drop_i) { - BATT_CHECK_GE(next_droppable, 0); - BATT_CHECK_LT(next_droppable, item_count); - - std::uniform_int_distribution pick_lower_bound{ - next_droppable, - item_count - (drops_remaining * 2 - 1), - }; - const u32 lower_bound_i = pick_lower_bound(rng); - - std::uniform_int_distribution pick_upper_bound{ - lower_bound_i + 1, - item_count - (drops_remaining * 2 - 2), - }; - const u32 upper_bound_i = pick_upper_bound(rng); - - BATT_CHECK_LT(lower_bound_i, upper_bound_i); - BATT_CHECK_GE(lower_bound_i, next_droppable); - - items_dropped += upper_bound_i - lower_bound_i; - - const usize live_count_before = leaf_filter.live().size(); - //----- --- -- - - - - - dropped_ranges.push_back(Interval{lower_bound_i, upper_bound_i}); - leaf_filter.drop_index_range(Interval{lower_bound_i, upper_bound_i}); - //----- --- -- - - - - - const usize live_count_after = leaf_filter.live().size(); - - if (lower_bound_i == 0) { - ASSERT_EQ(live_count_after, live_count_before); - } else { - ASSERT_EQ(live_count_after, live_count_before + 1); - } - - --drops_remaining; - next_droppable = upper_bound_i + 1; - } + std::tie(items_dropped, dropped_ranges) = + turtle_kv::testing::drop_n_disjoint_intervals_from(&leaf_filter, + drop_count, + Interval{0, item_count}, + rng); // Verify the number of expected live items. // @@ -333,8 +301,6 @@ TEST(TreePackedBlockedLeafPageTest, Random) batt::seq::for_each([&](const std::pair>& live_pair) { const auto [block_index, live_range] = live_pair; - // std::cerr << BATT_INSPECT(block_index) << BATT_INSPECT(live_range) << std::endl; - BATT_CHECK_GE(block_index, next_possible_block); BATT_CHECK_LT(block_index, packed_leaf.block_count()); BATT_CHECK_GE(live_range.lower_bound, next_possible_live) diff --git a/src/turtle_kv/util/piecewise_filter.test.cpp b/src/turtle_kv/util/piecewise_filter.test.cpp index fe4d3de..59b8d0b 100644 --- a/src/turtle_kv/util/piecewise_filter.test.cpp +++ b/src/turtle_kv/util/piecewise_filter.test.cpp @@ -15,9 +15,11 @@ #include #include +#include #include +#include #include #include #include @@ -40,6 +42,7 @@ using turtle_kv::PiecewiseFilter; using turtle_kv::Slice; using turtle_kv::Status; using turtle_kv::StatusOr; +using turtle_kv::testing::drop_n_disjoint_intervals_from; using turtle_kv::testing::RandomStringGenerator; using turtle_kv::drop_item_range; @@ -310,35 +313,35 @@ TEST(PiecewiseFilterTest, LiveSubranges) std::array, 1> init_live{{{0, 64}}}; const usize n_seeds = 10000000; - const usize n_drops = 10; + const usize n_drops = 32; const usize n_queries = 15; const usize first_seed = 0; + PiecewiseFilter filter; + + const auto query_as_bits = [&](Interval query) { + u64 bits = 0; + filter.live_subranges_of(query) | batt::seq::for_each([&bits](const Interval& live) { + bits |= mask_from_interval(live); + }); + return bits; + }; + for (usize seed_i = first_seed; seed_i < first_seed + n_seeds; ++seed_i) { std::default_random_engine rng{seed_i}; - PiecewiseFilter filter = - BATT_OK_RESULT_OR_PANIC(PiecewiseFilter::from_live(batt::as_slice(init_live))); + for (usize i = 0; i < n_drops; ++i) { + BATT_DEBUG_INFO(BATT_INSPECT(i) << BATT_INSPECT(seed_i)); - const auto query_as_bits = [&](Interval query) { - u64 bits = 0; - filter.live_subranges_of(query) | batt::seq::for_each([&bits](const Interval& live) { - bits |= mask_from_interval(live); - }); - return bits; - }; + filter = BATT_OK_RESULT_OR_PANIC(PiecewiseFilter::from_live(batt::as_slice(init_live))); + u64 filter_state = ~u64{0}; - u64 filter_state = ~u64{0}; + std::vector> dropped_ranges = + drop_n_disjoint_intervals_from(&filter, i, init_live[0], rng).second; - for (usize i = 0; i < n_drops; ++i) { - const Interval drop_interval = pick_interval(rng); - const u64 drop_mask = mask_from_interval(drop_interval); - filter_state &= ~drop_mask; - filter.drop_index_range(drop_interval); - - if constexpr (false) { - std::cerr << BATT_INSPECT(std::bitset<64>{filter_state}) << BATT_INSPECT(filter.dump()) - << std::endl; + for (const Interval& drop_interval : dropped_ranges) { + const u64 drop_mask = mask_from_interval(drop_interval); + filter_state &= ~drop_mask; } for (usize j = 0; j < n_queries; ++j) { diff --git a/src/turtle_kv/util/piecewise_filter.test.hpp b/src/turtle_kv/util/piecewise_filter.test.hpp new file mode 100644 index 0000000..32f71e6 --- /dev/null +++ b/src/turtle_kv/util/piecewise_filter.test.hpp @@ -0,0 +1,129 @@ +//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ +// +// Part of the TurtleKV Project, under Apache License v2.0. +// See https://www.apache.org/licenses/LICENSE-2.0 for license information. +// SPDX short identifier: Apache-2.0 +// +//+++++++++++-+-+--+----- --- -- - - - - + +#pragma once +#define TURTLE_KV_UTIL_PIECEWISE_FILTER_TEST_HPP + +#include "piecewise_filter.hpp" +#include "piecewise_filter_storage_model.concept.hpp" + +#include + +#include +#include + +#include +#include + +namespace turtle_kv { +namespace testing { + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +/** \brief Randomly drops `n` ranges within the specified live range of the passed filter. + * + * Requires that: + * - `drop_within` must be live in `filter` + * - `drop_within.size()` must be large enough to fit `n` disjoint intervals + * + * \return a pair of { total offset size dropped, vector of the dropped intervals } + */ +template ModelT> +inline std::pair>> drop_n_disjoint_intervals_from( + BasicPiecewiseFilter* filter, + usize n, + const Interval& drop_within, + Rng& rng) +{ + constexpr bool debug = false; + + if constexpr (debug) { + std::cerr << BATT_INSPECT(n) << std::endl; + } + + OffsetT dropped_total_size = 0; + std::vector> dropped_ranges; + + if (n == 0) { + return std::make_pair(dropped_total_size, dropped_ranges); + } + + BATT_CHECK_LE(n * 2 - 1, drop_within.size()); + BATT_CHECK_EQ(filter->live().empty(), false); + BATT_CHECK_EQ(filter->find_live_range(drop_within), drop_within); + + usize drops_remaining = n; + OffsetT next_droppable = drop_within.lower_bound; + const OffsetT live_lower_bound = filter->live().front().lower_bound; + const OffsetT live_upper_bound = filter->live().back().upper_bound; + + BATT_DEBUG_INFO(BATT_INSPECT(dropped_total_size) + << BATT_INSPECT_RANGE(dropped_ranges) << BATT_INSPECT(drop_within) + << BATT_INSPECT(n) << BATT_INSPECT(drops_remaining) + << BATT_INSPECT(next_droppable) << BATT_INSPECT(live_lower_bound) + << BATT_INSPECT(live_upper_bound)); + + if constexpr (debug) { + std::cerr << BATT_INSPECT_RANGE(filter->live()) << std::endl; + } + + for (usize drop_i = 0; drop_i < n; ++drop_i) { + BATT_CHECK_GE(next_droppable, 0); + BATT_CHECK_LT(next_droppable, drop_within.upper_bound); + + std::uniform_int_distribution pick_lower_bound{ + next_droppable, + drop_within.upper_bound - (drops_remaining * 2 - 1), + }; + const OffsetT lower_bound_i = pick_lower_bound(rng); + + std::uniform_int_distribution pick_upper_bound{ + lower_bound_i + 1, + drop_within.upper_bound - (drops_remaining * 2 - 2), + }; + const OffsetT upper_bound_i = pick_upper_bound(rng); + + BATT_CHECK_LT(lower_bound_i, upper_bound_i); + BATT_CHECK_GE(lower_bound_i, next_droppable); + + dropped_total_size += upper_bound_i - lower_bound_i; + + const usize live_count_before = filter->live().size(); + //----- --- -- - - - - + dropped_ranges.push_back(Interval{lower_bound_i, upper_bound_i}); + if constexpr (debug) { + std::cerr << " dropping: " << lower_bound_i << ".." << upper_bound_i << std::endl; + } + filter->drop_index_range(Interval{lower_bound_i, upper_bound_i}); + //----- --- -- - - - - + const usize live_count_after = filter->live().size(); + + if constexpr (debug) { + std::cerr << BATT_INSPECT_RANGE(filter->live()) << std::endl; + } + + if (lower_bound_i == live_lower_bound && upper_bound_i == live_upper_bound) { + BATT_CHECK_EQ(live_count_after + 1, live_count_before); + + } else if ((lower_bound_i == live_lower_bound && upper_bound_i != live_upper_bound) || + (upper_bound_i == live_upper_bound && lower_bound_i != live_lower_bound)) { + BATT_CHECK_EQ(live_count_after, live_count_before); + + } else { + BATT_CHECK_EQ(live_count_after, live_count_before + 1); + } + + --drops_remaining; + next_droppable = upper_bound_i + 1; + } + + return std::make_pair(dropped_total_size, dropped_ranges); +} + +} // namespace testing +} // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp index 6dd8701..5a105df 100644 --- a/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp +++ b/src/turtle_kv/util/piecewise_filter_storage_model.concept.hpp @@ -17,7 +17,6 @@ #include #include #include -#include #include namespace turtle_kv { From b122ff39d3339d3b328f071f10238c1ca8f6b972 Mon Sep 17 00:00:00 2001 From: Tony Astolfi Date: Sun, 28 Jun 2026 12:55:57 -0400 Subject: [PATCH 09/13] Test tuning. --- conan.lock | 4 ++-- conanfile.py | 2 +- src/turtle_kv/tree/in_memory_node.cpp | 2 ++ src/turtle_kv/tree/in_memory_node_merged_level.cpp | 2 ++ src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp | 2 +- src/turtle_kv/util/piecewise_filter.test.cpp | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/conan.lock b/conan.lock index 1cb73b8..d85e847 100644 --- a/conan.lock +++ b/conan.lock @@ -1,8 +1,8 @@ { "version": "0.5", "requires": [ - "abseil/20250127.0", - "artc/0.1.0", + "abseil/20260107.1", + "artc/0.2.1", "batteries/0.72.0", "boost/1.88.0", "bzip2/1.0.8", diff --git a/conanfile.py b/conanfile.py index 3474dd2..186c7f2 100644 --- a/conanfile.py +++ b/conanfile.py @@ -89,7 +89,7 @@ def requirements(self): } self.requires("abseil/[>=20260107.1]", **VISIBLE, **OVERRIDE) - self.requires("artc/[>=0.1.0 <1]") + self.requires("artc/[>=0.2.1 <1]") self.requires("batteries/[>=0.72.0 <1]", **VISIBLE, **OVERRIDE) self.requires("boost/[>=1.88.0 <2]", **VISIBLE, **OVERRIDE) self.requires("glog/[>=0.7.1 <1]", **VISIBLE) diff --git a/src/turtle_kv/tree/in_memory_node.cpp b/src/turtle_kv/tree/in_memory_node.cpp index 536e0c3..e0bcd71 100644 --- a/src/turtle_kv/tree/in_memory_node.cpp +++ b/src/turtle_kv/tree/in_memory_node.cpp @@ -15,6 +15,8 @@ #include #include +#include + #include #include diff --git a/src/turtle_kv/tree/in_memory_node_merged_level.cpp b/src/turtle_kv/tree/in_memory_node_merged_level.cpp index 7a67284..a28be9f 100644 --- a/src/turtle_kv/tree/in_memory_node_merged_level.cpp +++ b/src/turtle_kv/tree/in_memory_node_merged_level.cpp @@ -16,6 +16,8 @@ #include +#include + #include namespace turtle_kv { diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp index 03c926e..732154b 100644 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp @@ -66,7 +66,7 @@ using turtle_kv::ValueView; TEST(TreePackedBlockedLeafPageTest, Random) { const usize kFirstSeed = 0; - const usize kNumSeeds = 1000; + const usize kNumSeeds = 250; const usize kLastSeed = kFirstSeed + kNumSeeds; const usize kLeafPageSize = 1 * kMiB; const usize kNumPrefixes = 1000; diff --git a/src/turtle_kv/util/piecewise_filter.test.cpp b/src/turtle_kv/util/piecewise_filter.test.cpp index 59b8d0b..7fa32a8 100644 --- a/src/turtle_kv/util/piecewise_filter.test.cpp +++ b/src/turtle_kv/util/piecewise_filter.test.cpp @@ -312,7 +312,7 @@ TEST(PiecewiseFilterTest, LiveSubranges) std::array, 1> init_live{{{0, 64}}}; - const usize n_seeds = 10000000; + const usize n_seeds = 100000; const usize n_drops = 32; const usize n_queries = 15; const usize first_seed = 0; From 7d169af827d3fad1b73a79f9ad3c7a4f52ba720f Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Tue, 7 Jul 2026 10:32:29 -0400 Subject: [PATCH 10/13] Refactor with PackedPiecewiseFilter --- src/turtle_kv/tree/in_memory_node.cpp | 4 +- src/turtle_kv/tree/packed_node_page.cpp | 134 ++--------------- src/turtle_kv/tree/packed_node_page.hpp | 9 +- .../util/packed_piecewise_filter_view.hpp | 49 +++--- src/turtle_kv/util/piecewise_filter.hpp | 36 ++++- src/turtle_kv/util/piecewise_filter.ipp | 58 +++++++ .../util/piecewise_filter.live_subranges.hpp | 2 +- src/turtle_kv/util/piecewise_filter.test.cpp | 141 ++++++++++-------- src/turtle_kv/util/piecewise_filter.test.hpp | 118 +++++++++++++++ 9 files changed, 331 insertions(+), 220 deletions(-) diff --git a/src/turtle_kv/tree/in_memory_node.cpp b/src/turtle_kv/tree/in_memory_node.cpp index e0bcd71..a3a712b 100644 --- a/src/turtle_kv/tree/in_memory_node.cpp +++ b/src/turtle_kv/tree/in_memory_node.cpp @@ -118,8 +118,8 @@ using PackedSegment = PackedUpdateBuffer::Segment; segment.page_id_slot = llfs::PageIdSlot::from_page_id(packed_segment.leaf_page_id.unpack()); segment.active_pivots = packed_segment.active_pivots.unpack(); - BATT_ASSIGN_OK_RESULT(segment.filter, - packed_node.create_piecewise_filter(level_i, segment_i)); + segment.filter = + PiecewiseFilter{packed_node.get_packed_filter(level_i, segment_i)}; segment.check_invariants(__FILE__, __LINE__); } diff --git a/src/turtle_kv/tree/packed_node_page.cpp b/src/turtle_kv/tree/packed_node_page.cpp index 904b83e..c469d6f 100644 --- a/src/turtle_kv/tree/packed_node_page.cpp +++ b/src/turtle_kv/tree/packed_node_page.cpp @@ -222,9 +222,7 @@ StatusOr PackedNodePage::find_key(KeyQuery& query) const //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -PackedNodePage::UpdateBuffer::SegmentFilterData PackedNodePage::get_segment_filter_values( - usize level_i, - usize segment_i) const +PackedPiecewiseFilter PackedNodePage::get_packed_filter(usize level_i, usize segment_i) const { const usize i = [&]() -> usize { if (this->is_size_tiered()) { @@ -256,52 +254,9 @@ PackedNodePage::UpdateBuffer::SegmentFilterData PackedNodePage::get_segment_filt bool start_live = (segment.filter_start.value() & PackedNodePage::kSegmentStartsLive) != 0; - return PackedNodePage::UpdateBuffer::SegmentFilterData{ + return PackedPiecewiseFilter{PackedPiecewiseFilterStorage{ as_const_slice(packed_filters.data() + filter_start_i, packed_filters.data() + filter_end_i), - start_live}; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -StatusOr> PackedNodePage::create_piecewise_filter(usize level_i, - usize segment_i) const -{ - PackedNodePage::UpdateBuffer::SegmentFilterData filter_data = - this->get_segment_filter_values(level_i, segment_i); - - SmallVec, 64> live_ranges; - u32 i = 0; - - // If the first item at index 0 is live, add the corresponding interval first since the - // serialized version of the filter doesn't store index 0. - // - if (filter_data.start_is_live) { - if (filter_data.values.empty()) { - // Entire segment is live. - // - live_ranges.emplace_back(Interval{PiecewiseFilter::kMinLowerBound, - PiecewiseFilter::kMaxUpperBound}); - } else { - live_ranges.emplace_back( - Interval{PiecewiseFilter::kMinLowerBound, filter_data.values[i].value()}); - i++; - } - } - - for (; i + 1 < filter_data.values.size(); i += 2) { - live_ranges.emplace_back( - Interval{filter_data.values[i].value(), filter_data.values[i + 1].value()}); - } - - // If there's one unpaired value left, it's a lower_bound whose upper_bound (kMaxUpperBound) - // was omitted. - // - if (i < filter_data.values.size()) { - live_ranges.emplace_back( - Interval{filter_data.values[i].value(), PiecewiseFilter::kMaxUpperBound}); - } - - return PiecewiseFilter::from_live(as_slice(live_ranges)); + start_live}}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -339,7 +294,11 @@ StatusOr PackedNodePage::UpdateBuffer::Segment::load_leaf_page bool PackedNodePage::UpdateBuffer::Segment::is_index_filtered(const SegmentedLevel& level, u32 index) const { - return !(this->live_lower_bound(level, index) == index); + const usize segment_i = std::distance(level.segments_slice.begin(), this); + + PackedPiecewiseFilter filter = level.packed_node_->get_packed_filter(level.level_i_, segment_i); + + return !filter.live_at_index(index); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -348,34 +307,10 @@ u32 PackedNodePage::UpdateBuffer::Segment::live_lower_bound(const SegmentedLevel u32 item_i) const { const usize segment_i = std::distance(level.segments_slice.begin(), this); - PackedNodePage::UpdateBuffer::SegmentFilterData filter_data = - level.packed_node_->get_segment_filter_values(level.level_i_, segment_i); - - const Slice filter_values = filter_data.values; - - if (filter_data.values.empty()) { - BATT_CHECK(filter_data.start_is_live); - return item_i; - } - - auto iter = std::upper_bound(filter_values.begin(), filter_values.end(), item_i); - - usize previous_cut_points = std::distance(filter_data.values.begin(), iter); - - bool is_live = (previous_cut_points % 2 == 0) == filter_data.start_is_live; - - // If we're already in an unfiltered region, just return the index. Otherwise, our upper bound - // is the next unfiltered index. - // - if (is_live) { - return item_i; - } - - if (iter != filter_values.end()) { - return iter->value(); - } - - return PiecewiseFilter::kMaxUpperBound; + + PackedPiecewiseFilter filter = level.packed_node_->get_packed_filter(level.level_i_, segment_i); + + return filter.live_lower_bound(item_i); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - @@ -384,48 +319,11 @@ Interval PackedNodePage::UpdateBuffer::Segment::get_live_item_range( const SegmentedLevel& level, Interval i) const { - u32 start_i = i.lower_bound; - u32 end_i = i.upper_bound; - - BATT_CHECK_LT(start_i, end_i); - const usize segment_i = std::distance(level.segments_slice.begin(), this); - PackedNodePage::UpdateBuffer::SegmentFilterData filter_data = - level.packed_node_->get_segment_filter_values(level.level_i_, segment_i); - - const Slice filter_values = filter_data.values; - - if (filter_data.values.empty()) { - BATT_CHECK(filter_data.start_is_live); - return i; - } - - auto iter = std::upper_bound(filter_values.begin(), filter_values.end(), start_i); - - usize previous_cut_points = std::distance(filter_data.values.begin(), iter); - - bool is_live = (previous_cut_points % 2 == 0) == filter_data.start_is_live; - - if (!is_live) { - if (iter == filter_values.end()) { - return Interval{end_i, end_i}; - } - - start_i = iter->value(); - if (start_i >= end_i) { - return Interval{end_i, end_i}; - } - - ++iter; - } - - if (iter != filter_values.end()) { - end_i = std::min(end_i, iter->value()); - } - - BATT_CHECK_LT(start_i, end_i) << BATT_INSPECT(i); - - return Interval{start_i, end_i}; + + PackedPiecewiseFilter filter = level.packed_node_->get_packed_filter(level.level_i_, segment_i); + + return filter.find_live_range(i); } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // diff --git a/src/turtle_kv/tree/packed_node_page.hpp b/src/turtle_kv/tree/packed_node_page.hpp index 1e7cc66..5f4e92e 100644 --- a/src/turtle_kv/tree/packed_node_page.hpp +++ b/src/turtle_kv/tree/packed_node_page.hpp @@ -121,11 +121,6 @@ struct PackedNodePage { struct UpdateBuffer { struct SegmentedLevel; - struct SegmentFilterData { - Slice values; - bool start_is_live; - }; - struct Segment { llfs::PackedPageId leaf_page_id; // +8 -> 8 PackedActivePivotsSet64 active_pivots; // +8 -> 16 @@ -387,9 +382,7 @@ struct PackedNodePage { StatusOr find_key_in_level(usize level_i, KeyQuery& query, i32 key_pivot_i) const; - UpdateBuffer::SegmentFilterData get_segment_filter_values(usize level_i, usize segment_i) const; - - StatusOr> create_piecewise_filter(usize level_i, usize segment_i) const; + PackedPiecewiseFilter get_packed_filter(usize level_i, usize segment_i) const; //----- --- -- - - - - diff --git a/src/turtle_kv/util/packed_piecewise_filter_view.hpp b/src/turtle_kv/util/packed_piecewise_filter_view.hpp index 069cc11..990088f 100644 --- a/src/turtle_kv/util/packed_piecewise_filter_view.hpp +++ b/src/turtle_kv/util/packed_piecewise_filter_view.hpp @@ -16,6 +16,7 @@ #include #include +#include #include @@ -50,7 +51,7 @@ namespace turtle_kv { * Live Intervals: {[10, 20), [30, 40), [50, +inf)} * Packed: start_is_live=0, {10, 20, 30, 40, 50} */ -class PackedPiecewiseFilterView +class PackedPiecewiseFilterStorage { public: //----- --- -- - - - - @@ -73,35 +74,35 @@ class PackedPiecewiseFilterView // Forward-declaration; defined below. // - friend const Slice& as_const_slice(const PackedPiecewiseFilterView& view); + friend const Slice& as_const_slice(const PackedPiecewiseFilterStorage& view); //----- --- -- - - - - - /** \brief Constructs an PackedPiecewiseFilterView representing the live interval [0, +inf). + /** \brief Constructs an PackedPiecewiseFilterStorage representing the live interval [0, +inf). */ - PackedPiecewiseFilterView() = default; + PackedPiecewiseFilterStorage() = default; - /** \brief Destructs the PackedPiecewiseFilterView. + /** \brief Destructs the PackedPiecewiseFilterStorage. */ - ~PackedPiecewiseFilterView() = default; + ~PackedPiecewiseFilterStorage() = default; - /** \brief PackedPiecewiseFilterView is copy constructible. + /** \brief PackedPiecewiseFilterStorage is copy constructible. */ - PackedPiecewiseFilterView(const PackedPiecewiseFilterView&) = default; + PackedPiecewiseFilterStorage(const PackedPiecewiseFilterStorage&) = default; - /** \brief PackedPiecewiseFilterView is copy assignable. + /** \brief PackedPiecewiseFilterStorage is copy assignable. */ - PackedPiecewiseFilterView& operator=(const PackedPiecewiseFilterView&) = default; + PackedPiecewiseFilterStorage& operator=(const PackedPiecewiseFilterStorage&) = default; - /** \brief Constructs PackedPiecewiseFilterView from the packed data in the arguments. + /** \brief Constructs PackedPiecewiseFilterStorage from the packed data in the arguments. * * See the class-level description for details on what `values` and `start_is_live` represent. */ - explicit PackedPiecewiseFilterView(const Slice& values, + explicit PackedPiecewiseFilterStorage(const Slice& values, bool start_is_live) noexcept : values_{values} , implicit_first_{start_is_live ? 1 : 0} - , size_{(this->implicit_first_ + this->values_.size() + 1) & ~i32{1}} + , size_{(this->implicit_first_ + BATT_CHECKED_CAST(i32, this->values_.size()) + 1) / 2} { } @@ -148,7 +149,7 @@ class PackedPiecewiseFilterView // /** \brief Returns a const reference to the stored values referenced by `view`. */ -inline const Slice& as_const_slice(const PackedPiecewiseFilterView& view) +inline const Slice& as_const_slice(const PackedPiecewiseFilterStorage& view) { return view.values_; } @@ -157,9 +158,9 @@ inline const Slice& as_const_slice(const PackedPiecewiseFilter // /** \brief Read-only, random access iterator over the live intervals of a packed piecewise filter. */ -class PackedPiecewiseFilterView::const_iterator +class PackedPiecewiseFilterStorage::const_iterator : public boost::iterator_facade< // - PackedPiecewiseFilterView::const_iterator, // <- Derived + PackedPiecewiseFilterStorage::const_iterator, // <- Derived Interval, // <- Value std::random_access_iterator_tag, // <- CategoryOrTraversal Interval, // <- Reference @@ -184,7 +185,7 @@ class PackedPiecewiseFilterView::const_iterator * * `view` must remain in-scope while this object exists. */ - const_iterator(const PackedPiecewiseFilterView* view, isize pos) noexcept : view_{view}, pos_{pos} + const_iterator(const PackedPiecewiseFilterStorage* view, isize pos) noexcept : view_{view}, pos_{pos} { } @@ -239,7 +240,7 @@ class PackedPiecewiseFilterView::const_iterator private: /** \brief Pointer to the filter view over which we are iterating. */ - const PackedPiecewiseFilterView* view_; + const PackedPiecewiseFilterStorage* view_; /** \brief The (logical) position of this iterator within `view_`. */ @@ -248,35 +249,35 @@ class PackedPiecewiseFilterView::const_iterator //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline auto PackedPiecewiseFilterView::begin() const noexcept -> const_iterator +inline auto PackedPiecewiseFilterStorage::begin() const noexcept -> const_iterator { return const_iterator{this, 0}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline auto PackedPiecewiseFilterView::end() const noexcept -> const_iterator +inline auto PackedPiecewiseFilterStorage::end() const noexcept -> const_iterator { return const_iterator{this, static_cast(this->size())}; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline usize PackedPiecewiseFilterView::size() const noexcept +inline usize PackedPiecewiseFilterStorage::size() const noexcept { return this->size_; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline bool PackedPiecewiseFilterView::empty() const noexcept +inline bool PackedPiecewiseFilterStorage::empty() const noexcept { return this->size_ == 0; } //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // -inline Interval PackedPiecewiseFilterView::operator[](isize i) const noexcept +inline Interval PackedPiecewiseFilterStorage::operator[](isize i) const noexcept { // Cached for brevity below. // @@ -300,6 +301,6 @@ inline Interval PackedPiecewiseFilterView::operator[](isize i) const noexce //=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -static_assert(PiecewiseFilterStorageModel); +static_assert(PiecewiseFilterStorageModel); } // namespace turtle_kv diff --git a/src/turtle_kv/util/piecewise_filter.hpp b/src/turtle_kv/util/piecewise_filter.hpp index 16a7cfc..9f22ceb 100644 --- a/src/turtle_kv/util/piecewise_filter.hpp +++ b/src/turtle_kv/util/piecewise_filter.hpp @@ -9,6 +9,7 @@ #pragma once #define TURTLE_KV_UTIL_PIECEWISE_FILTER_HPP +#include "packed_piecewise_filter_view.hpp" #include "piecewise_filter_storage_model.concept.hpp" #include @@ -62,7 +63,19 @@ class BasicPiecewiseFilter : private ModelT /** \brief Constructs a default instance of a PiecewiseFilter object, initialized with no item * range and filtered items. */ - BasicPiecewiseFilter() noexcept; + BasicPiecewiseFilter() noexcept + requires PiecewiseFilterMutableStorageModel; + + /** \brief Constructs a BasicPiecewiseFilter directly from a storage model instance. + */ + explicit BasicPiecewiseFilter(const ModelT& model) noexcept; + + /** \brief Constructs a BasicPiecewiseFilter by copying live intervals from a filter with a + * different storage model. + */ + template OtherModelT> + explicit BasicPiecewiseFilter(const BasicPiecewiseFilter& other) + requires PiecewiseFilterMutableStorageModel; //+++++++++++-+-+--+----- --- -- - - - - @@ -105,7 +118,8 @@ class BasicPiecewiseFilter : private ModelT /** \brief Returns a view of the live item intervals. */ - Slice> live() const; + Slice> live() const + requires PiecewiseFilterMutableStorageModel; /** \brief Merges two filters in place, taking the union of the live intervals. */ @@ -117,6 +131,22 @@ class BasicPiecewiseFilter : private ModelT */ LiveSubranges live_subranges_of(Interval i) const; + /** \brief Returns an iterator to the first live interval. + */ + ConstIterator begin() const; + + /** \brief Returns an iterator past the last live interval. + */ + ConstIterator end() const; + + /** \brief Returns the number of live intervals. + */ + usize size() const; + + /** \brief Returns true iff there are no live intervals. + */ + bool empty() const; + /** \brief Validate the state of the live intervals. */ bool check_invariants() const; @@ -139,6 +169,8 @@ class BasicPiecewiseFilter : private ModelT template using PiecewiseFilter = BasicPiecewiseFilter, 64>>; +using PackedPiecewiseFilter = BasicPiecewiseFilter; + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template diff --git a/src/turtle_kv/util/piecewise_filter.ipp b/src/turtle_kv/util/piecewise_filter.ipp index b892275..3848365 100644 --- a/src/turtle_kv/util/piecewise_filter.ipp +++ b/src/turtle_kv/util/piecewise_filter.ipp @@ -36,10 +36,35 @@ BasicPiecewiseFilter::from_live(const Slice ModelT> BasicPiecewiseFilter::BasicPiecewiseFilter() noexcept + requires PiecewiseFilterMutableStorageModel : ModelT{{Interval{Self::kMinLowerBound, Self::kMaxUpperBound}}} { } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +BasicPiecewiseFilter::BasicPiecewiseFilter(const ModelT& model) noexcept + : ModelT{model} +{ +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +template OtherModelT> +BasicPiecewiseFilter::BasicPiecewiseFilter( + const BasicPiecewiseFilter& other) + requires PiecewiseFilterMutableStorageModel + : ModelT{} +{ + this->live_().clear(); + + for (auto iter = other.begin(); iter != other.end(); ++iter) { + this->live_().insert(this->live_().end(), *iter); + } +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template ModelT> @@ -204,10 +229,43 @@ Interval BasicPiecewiseFilter::drop_index_range(Interv // template ModelT> Slice> BasicPiecewiseFilter::live() const + requires PiecewiseFilterMutableStorageModel { return as_const_slice(this->live_()); } +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +auto BasicPiecewiseFilter::begin() const -> ConstIterator +{ + return this->live_().begin(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +auto BasicPiecewiseFilter::end() const -> ConstIterator +{ + return this->live_().end(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +usize BasicPiecewiseFilter::size() const +{ + return this->live_().size(); +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +template ModelT> +bool BasicPiecewiseFilter::empty() const +{ + return this->live_().empty(); +} + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // template ModelT> diff --git a/src/turtle_kv/util/piecewise_filter.live_subranges.hpp b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp index c2bf50c..a22a0b1 100644 --- a/src/turtle_kv/util/piecewise_filter.live_subranges.hpp +++ b/src/turtle_kv/util/piecewise_filter.live_subranges.hpp @@ -22,7 +22,7 @@ template ModelT> class BasicPiecewiseFilter::LiveSubranges { public: - using Iterator = PiecewiseFilter::ConstIterator; + using Iterator = typename BasicPiecewiseFilter::ConstIterator; using Item = Interval; //+++++++++++-+-+--+----- --- -- - - - - diff --git a/src/turtle_kv/util/piecewise_filter.test.cpp b/src/turtle_kv/util/piecewise_filter.test.cpp index 7fa32a8..7aa134e 100644 --- a/src/turtle_kv/util/piecewise_filter.test.cpp +++ b/src/turtle_kv/util/piecewise_filter.test.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -42,8 +43,14 @@ using turtle_kv::PiecewiseFilter; using turtle_kv::Slice; using turtle_kv::Status; using turtle_kv::StatusOr; +using turtle_kv::testing::build_filter_with_random_drops; using turtle_kv::testing::drop_n_disjoint_intervals_from; +using turtle_kv::testing::get_packed_filter_from_data; +using turtle_kv::testing::PackedFilterData; +using turtle_kv::testing::pack_in_memory_filter; +using turtle_kv::testing::RandomDropResult; using turtle_kv::testing::RandomStringGenerator; +using turtle_kv::testing::verify_filter_queries; using turtle_kv::drop_item_range; @@ -52,6 +59,10 @@ using llfs::KeyRangeOrder; using batt::mask_from_interval; using batt::StableStringStore; +using turtle_kv::PackedPiecewiseFilter; +using turtle_kv::PackedPiecewiseFilterStorage; + + //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - // TEST(PiecewiseFilterTest, InvalidFilterTest) @@ -84,89 +95,68 @@ TEST(PiecewiseFilterTest, InvalidFilterTest) // TEST(PiecewiseFilterTest, QueryTest) { - const usize num_items = 10000; + const u32 num_items = 10000; - for (usize seed = 0; seed < 100; ++seed) { + for (u32 seed = 0; seed < 100; ++seed) { std::default_random_engine rng{seed}; - PiecewiseFilter filter; - EXPECT_TRUE(filter.check_invariants()); + auto [filter, live_items] = build_filter_with_random_drops(num_items, rng); - // All items start live. - // - std::set live_items; - for (usize i = 0; i < num_items; ++i) { - live_items.insert(i); - } + EXPECT_TRUE(filter.check_invariants()); - // Drop random intervals. - // - std::uniform_int_distribution pick_num_dropped{100, num_items / 2}; - usize num_intervals_dropped = pick_num_dropped(rng); - for (usize i = 0; i < num_intervals_dropped; ++i) { - std::uniform_int_distribution pick_interval_start{0, num_items - 1}; - usize start_i = pick_interval_start(rng); + verify_filter_queries(filter, live_items, num_items, seed, rng); + } +} - std::uniform_int_distribution pick_interval_end{start_i, num_items}; - usize end_i = pick_interval_end(rng); +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - +// +TEST(PiecewiseFilterTest, PackedQueryTest) +{ + const u32 num_items = 10000; - for (usize j = start_i; j < end_i; ++j) { - live_items.erase(j); - } + for (u32 seed = 0; seed < 100; ++seed) { + std::default_random_engine rng{seed}; - Interval new_dropped = filter.drop_index_range(Interval{start_i, end_i}); - EXPECT_LE(new_dropped.lower_bound, start_i) << BATT_INSPECT(seed); - EXPECT_GE(new_dropped.upper_bound, end_i) << BATT_INSPECT(seed); - } + auto [filter, live_items] = build_filter_with_random_drops(num_items, rng); EXPECT_TRUE(filter.check_invariants()); - // Test live_at_index + // Pack the filter and construct a PackedPiecewiseFilter. // - for (usize i = 0; i < num_items; ++i) { - bool expected_live = live_items.count(i) > 0; - bool actual_live = filter.live_at_index(i); - EXPECT_EQ(actual_live, expected_live) << BATT_INSPECT(seed) << BATT_INSPECT(i); - } + PackedFilterData packed_data = pack_in_memory_filter(filter); + PackedPiecewiseFilter packed_filter = get_packed_filter_from_data(packed_data); - // Test live_lower_bound + // Verify the packed filter has the same number of live intervals. // - for (usize i = 0; i < num_items; ++i) { - auto iter = live_items.lower_bound(i); - usize expected = (iter != live_items.end()) ? *iter : num_items; - usize actual = filter.live_lower_bound(i); - EXPECT_EQ(actual, expected) << BATT_INSPECT(seed) << BATT_INSPECT(i); - } + EXPECT_EQ(packed_filter.size(), filter.size()) << BATT_INSPECT(seed); - // Test find_live_range + // Verify the packed filter produces identical intervals. // - for (usize i = 0; i < 100; ++i) { - std::uniform_int_distribution pick_interval_start{0, num_items - 1}; - usize start_i = pick_interval_start(rng); - - std::uniform_int_distribution pick_interval_end{start_i, num_items}; - usize end_i = pick_interval_end(rng); - - auto iter = live_items.lower_bound(start_i); - Interval expected_range; - - if (iter == live_items.end() || *iter >= end_i) { - expected_range = Interval{end_i, end_i}; - } else { - usize first = *iter; - usize last = first + 1; - auto next = std::next(iter); - - while (next != live_items.end() && *next < end_i && *next == last) { - ++last; - ++next; - } - - expected_range = Interval{first, last}; + { + auto mutable_iter = filter.begin(); + auto packed_iter = packed_filter.begin(); + while (mutable_iter != filter.end() && packed_iter != packed_filter.end()) { + EXPECT_EQ(*mutable_iter, *packed_iter) << BATT_INSPECT(seed); + ++mutable_iter; + ++packed_iter; } + EXPECT_EQ(mutable_iter, filter.end()) << BATT_INSPECT(seed); + EXPECT_EQ(packed_iter, packed_filter.end()) << BATT_INSPECT(seed); + } - Interval actual_range = filter.find_live_range(Interval{start_i, end_i}); - EXPECT_EQ(actual_range, expected_range) << BATT_INSPECT(seed); + // Verify queries produce the same results as the in-memory filter. + // + verify_filter_queries(packed_filter, live_items, num_items, seed, rng); + + // Converting packed back to in-memory should produce identical filter. + // + PiecewiseFilter converted_filter{packed_filter}; + EXPECT_TRUE(converted_filter.check_invariants()); + Slice> original_live = filter.live(); + Slice> converted_live = converted_filter.live(); + ASSERT_EQ(original_live.size(), converted_live.size()) << BATT_INSPECT(seed); + for (usize i = 0; i < original_live.size(); ++i) { + EXPECT_EQ(original_live[i], converted_live[i]) << BATT_INSPECT(seed) << BATT_INSPECT(i); } } } @@ -344,6 +334,20 @@ TEST(PiecewiseFilterTest, LiveSubranges) filter_state &= ~drop_mask; } + // Also test via PackedPiecewiseFilter. + // + PackedFilterData packed_data = pack_in_memory_filter(filter); + PackedPiecewiseFilter packed_filter = get_packed_filter_from_data(packed_data); + + const auto packed_query_as_bits = [&](Interval query) { + u64 bits = 0; + packed_filter.live_subranges_of(query) | + batt::seq::for_each([&bits](const Interval& live) { + bits |= mask_from_interval(live); + }); + return bits; + }; + for (usize j = 0; j < n_queries; ++j) { const Interval query_interval = pick_interval(rng); const u64 query_mask = mask_from_interval(query_interval); @@ -353,6 +357,13 @@ TEST(PiecewiseFilterTest, LiveSubranges) ASSERT_EQ(std::bitset<64>{expected_bits}, std::bitset<64>{actual_bits}) << BATT_INSPECT(seed_i) << BATT_INSPECT(query_interval) << BATT_INSPECT(query_interval.size()) << BATT_INSPECT(std::bitset<64>{query_mask}); + + const u64 packed_actual_bits = packed_query_as_bits(query_interval); + + ASSERT_EQ(std::bitset<64>{expected_bits}, std::bitset<64>{packed_actual_bits}) + << "PackedPiecewiseFilter mismatch: " << BATT_INSPECT(seed_i) + << BATT_INSPECT(query_interval) << BATT_INSPECT(query_interval.size()) + << BATT_INSPECT(std::bitset<64>{query_mask}); } } } diff --git a/src/turtle_kv/util/piecewise_filter.test.hpp b/src/turtle_kv/util/piecewise_filter.test.hpp index 32f71e6..f4058e5 100644 --- a/src/turtle_kv/util/piecewise_filter.test.hpp +++ b/src/turtle_kv/util/piecewise_filter.test.hpp @@ -125,5 +125,123 @@ inline std::pair>> drop_n_disjoint_interv return std::make_pair(dropped_total_size, dropped_ranges); } +struct PackedFilterData { + std::vector values; + bool start_is_live; +}; + +inline PackedFilterData pack_in_memory_filter(const PiecewiseFilter& filter) +{ + PackedFilterData data; + data.start_is_live = false; + Slice> live = filter.live(); + + if (live.empty()) { + return data; + } + + data.start_is_live = (live[0].lower_bound == PiecewiseFilter::kMinLowerBound); + + for (const Interval& range : live) { + if (range.lower_bound != PiecewiseFilter::kMinLowerBound) { + data.values.push_back(range.lower_bound); + } + if (range.upper_bound != PiecewiseFilter::kMaxUpperBound) { + data.values.push_back(range.upper_bound); + } + } + + return data; +} + +inline PackedPiecewiseFilter get_packed_filter_from_data(const PackedFilterData& data) +{ + return PackedPiecewiseFilter{PackedPiecewiseFilterStorage{ + batt::as_const_slice(data.values), data.start_is_live}}; +} + +//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - + +struct RandomDropResult { + PiecewiseFilter filter; + std::set live_items; +}; + +inline RandomDropResult build_filter_with_random_drops(u32 num_items, std::default_random_engine& rng) +{ + RandomDropResult result; + for (u32 i = 0; i < num_items; ++i) { + result.live_items.insert(i); + } + + std::uniform_int_distribution pick_num_dropped{100, num_items / 2}; + u32 num_intervals_dropped = pick_num_dropped(rng); + for (u32 i = 0; i < num_intervals_dropped; ++i) { + std::uniform_int_distribution pick_interval_start{0, num_items - 1}; + u32 start_i = pick_interval_start(rng); + + std::uniform_int_distribution pick_interval_end{start_i, num_items}; + u32 end_i = pick_interval_end(rng); + + for (u32 j = start_i; j < end_i; ++j) { + result.live_items.erase(j); + } + + result.filter.drop_index_range(Interval{start_i, end_i}); + } + + return result; +} + +template +inline void verify_filter_queries(const FilterT& filter, + const std::set& live_items, + u32 num_items, + u32 seed, + std::default_random_engine& rng) +{ + for (u32 i = 0; i < num_items; ++i) { + bool expected_live = live_items.count(i) > 0; + bool actual_live = filter.live_at_index(i); + EXPECT_EQ(actual_live, expected_live) << BATT_INSPECT(seed) << BATT_INSPECT(i); + } + + for (u32 i = 0; i < num_items; ++i) { + auto iter = live_items.lower_bound(i); + u32 expected = (iter != live_items.end()) ? *iter : num_items; + u32 actual = filter.live_lower_bound(i); + EXPECT_EQ(actual, expected) << BATT_INSPECT(seed) << BATT_INSPECT(i); + } + + for (u32 i = 0; i < 100; ++i) { + std::uniform_int_distribution pick_interval_start{0, num_items - 1}; + u32 start_i = pick_interval_start(rng); + + std::uniform_int_distribution pick_interval_end{start_i, num_items}; + u32 end_i = pick_interval_end(rng); + + auto iter = live_items.lower_bound(start_i); + Interval expected_range; + + if (iter == live_items.end() || *iter >= end_i) { + expected_range = Interval{end_i, end_i}; + } else { + u32 first = *iter; + u32 last = first + 1; + auto next = std::next(iter); + + while (next != live_items.end() && *next < end_i && *next == last) { + ++last; + ++next; + } + + expected_range = Interval{first, last}; + } + + Interval actual_range = filter.find_live_range(Interval{start_i, end_i}); + EXPECT_EQ(actual_range, expected_range) << BATT_INSPECT(seed) << BATT_INSPECT(i); + } +} + } // namespace testing } // namespace turtle_kv From 62b6fa5528518c80cefaa18c4f06200fc1f6272c Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Tue, 18 Aug 2026 18:01:25 -0400 Subject: [PATCH 11/13] Cherry pick PiecewiseFilter refactor. --- conan.lock | 1 - conanfile.py | 1 - src/CMakeLists.txt | 1 - src/turtle_kv/core/packed_key_value_slot.hpp | 5 - .../core/packed_key_value_slot_slice.hpp | 36 -- .../leaf/blocked_leaf_page_loader.concept.hpp | 30 -- .../tree/leaf/packed_blocked_leaf_page.cpp | 49 --- .../tree/leaf/packed_blocked_leaf_page.hpp | 337 ------------------ .../tree/leaf/packed_blocked_leaf_page.ipp | 316 ---------------- ...packed_blocked_leaf_page.item_iterator.hpp | 183 ---------- ..._blocked_leaf_page.sharded_live_ranges.hpp | 56 --- ..._blocked_leaf_page.sharded_live_ranges.ipp | 170 --------- .../leaf/packed_blocked_leaf_page.test.cpp | 332 ----------------- src/turtle_kv/tree/leaf/packed_leaf_block.hpp | 170 --------- src/turtle_kv/tree/leaf/packed_leaf_block.ipp | 162 --------- .../tree/leaf/packed_leaf_block.iterator.hpp | 104 ------ .../tree/leaf/packed_leaf_block.test.cpp | 173 --------- .../tree/leaf/packed_leaf_block_stats.hpp | 38 -- .../tree/leaf/packed_leaf_block_stats.ipp | 56 --- .../tree/packed_leaf_block_scanner.hpp | 75 ---- 20 files changed, 2295 deletions(-) delete mode 100644 src/turtle_kv/core/packed_key_value_slot_slice.hpp delete mode 100644 src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp delete mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp delete mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp delete mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp delete mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp delete mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp delete mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp delete mode 100644 src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp delete mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block.hpp delete mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block.ipp delete mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp delete mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp delete mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp delete mode 100644 src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp delete mode 100644 src/turtle_kv/tree/packed_leaf_block_scanner.hpp diff --git a/conan.lock b/conan.lock index d85e847..ebdb611 100644 --- a/conan.lock +++ b/conan.lock @@ -2,7 +2,6 @@ "version": "0.5", "requires": [ "abseil/20260107.1", - "artc/0.2.1", "batteries/0.72.0", "boost/1.88.0", "bzip2/1.0.8", diff --git a/conanfile.py b/conanfile.py index 186c7f2..aa69a04 100644 --- a/conanfile.py +++ b/conanfile.py @@ -89,7 +89,6 @@ def requirements(self): } self.requires("abseil/[>=20260107.1]", **VISIBLE, **OVERRIDE) - self.requires("artc/[>=0.2.1 <1]") self.requires("batteries/[>=0.72.0 <1]", **VISIBLE, **OVERRIDE) self.requires("boost/[>=1.88.0 <2]", **VISIBLE, **OVERRIDE) self.requires("glog/[>=0.7.1 <1]", **VISIBLE) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ab268b2..8631e97 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,7 +34,6 @@ endif () target_link_libraries( turtle_kv PUBLIC - artc::artc abseil::abseil batteries::batteries boost::boost diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index 342f315..ce42137 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -142,11 +142,6 @@ inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr* }; } -inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr& p_slot_ref) noexcept -{ - return to_key_value_slot_ref(std::addressof(p_slot_ref)); -} - inline PackedKeyValueSlotRef to_key_value_slot_ref(const ConstBuffer& slot_buffer) noexcept { return PackedKeyValueSlotRef{ diff --git a/src/turtle_kv/core/packed_key_value_slot_slice.hpp b/src/turtle_kv/core/packed_key_value_slot_slice.hpp deleted file mode 100644 index 7f2ee39..0000000 --- a/src/turtle_kv/core/packed_key_value_slot_slice.hpp +++ /dev/null @@ -1,36 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_CORE_PACKED_KEY_VALUE_SLOT_SLICE_HPP - -#include - -#include - -#include - -namespace turtle_kv { - -using PackedKeyValueSlotSlice = std::variant< // - Slice, - Slice>; - -struct ToPackedKeyValueSlotSlice { - PackedKeyValueSlotSlice operator()(const Slice& ref_slice) - { - return PackedKeyValueSlotSlice{ref_slice}; - } - - PackedKeyValueSlotSlice operator()(const Slice& ptr_slice) - { - return PackedKeyValueSlotSlice{ptr_slice}; - } -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp b/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp deleted file mode 100644 index f9832cf..0000000 --- a/src/turtle_kv/tree/leaf/blocked_leaf_page_loader.concept.hpp +++ /dev/null @@ -1,30 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_BLOCKED_LEAF_PAGE_LOADER_CONCEPT_HPP - -#include - -#include - -#include - -#include - -namespace turtle_kv { - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -template -concept BlockedLeafPageLoader = requires(T& loader, llfs::PageId page_id, BlockIndex block_i) { - loader.release_block(page_id, block_i); - { loader.load_block(page_id, block_i) } -> std::convertible_to>; -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp deleted file mode 100644 index 5e61632..0000000 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.cpp +++ /dev/null @@ -1,49 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#include "packed_blocked_leaf_page.hpp" -// - -namespace turtle_kv { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -/*static*/ usize PackedBlockedLeafPage::estimate_capacity(usize leaf_size, - usize block_size, - usize max_key_size, - usize max_edit_size) noexcept -{ - const usize space_after_header = - leaf_size - (sizeof(llfs::PackedPageHeader) + sizeof(PackedBlockedLeafPage)); - - const usize max_block_count = space_after_header / block_size; - - const usize block_starts_size = - sizeof(llfs::PackedArray) + sizeof(little_u32) * max_block_count; - - const usize space_after_block_starts = space_after_header - block_starts_size; - - const usize max_art_size = max_key_size * max_block_count * 2; - - const usize space_after_art = space_after_block_starts - max_art_size; - - BATT_CHECK_EQ(batt::bit_count(block_size), 1) << "Leaf block_size must be a power of 2"; - const usize space_for_blocks = space_after_art & ~(block_size - 1); - const usize block_count = space_for_blocks / block_size; - - const usize max_wasted_per_block = max_edit_size - 1; - const usize min_block_capacity = PackedLeafBlock::capacity(block_size) - max_wasted_per_block; - - const usize final_estimate = block_count * min_block_capacity; - - BATT_CHECK_GT(leaf_size, final_estimate); - - return final_estimate; -} - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp deleted file mode 100644 index b043193..0000000 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.hpp +++ /dev/null @@ -1,337 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_HPP - -#include "packed_leaf_block.hpp" -#include "packed_leaf_block.iterator.hpp" - -#include -#include - -#include - -#include - -#include - -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include - -#include - -namespace turtle_kv { - -// Forward-declaration. -// -struct PackedBlockedLeafPage; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -/** \brief Packs a blocked leaf page with the passed block size, containing the passed key/value - * pairs, into the passed buffer. - */ -template -StatusOr pack_blocked_leaf_page(const usize block_size, - const ItemRangeT& src_items, - const MutableBuffer& dst_buffer) noexcept; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief Header for a packed leaf page with blocked structure. - */ -struct PackedBlockedLeafPage // -{ - /** \brief Must be the first 8 bytes of the header. \see PackedBlockedLeagPage::magic - */ - static constexpr u64 kMagic = 0x6456beb7f9558445ull; - - //+++++++++++-+-+--+----- --- -- - - - - - - using BlockIterator = PackedLeafBlock::Iterator; - class ItemIterator; - - using BlockItemsSeq = PackedLeafBlock::BlockItemsSeq; - - struct ItemsSeqFromBlock { - BlockItemsSeq operator()(const PackedLeafBlock& block) const - { - return block.items_seq(); - } - }; - - using BlocksSeq = batt::SubRangeSeq>; - using ItemsSeq = batt::seq::Flatten>; - - struct SlotSliceFromBlock { - PackedKeyValueSlotSlice operator()(const PackedLeafBlock& block) const - { - return {block.items_slice()}; - } - }; - - using SlotSliceSeq = batt::seq::Map; - - template FilterModelT> - class ShardedLiveRanges; - - class HeaderShardView; - - //+++++++++++-+-+--+----- --- -- - - - - - - template - static usize packed_edit_size(const EditT& edit) noexcept - { - return PackedLeafBlock::packed_edit_size(edit); - } - - static usize estimate_capacity(usize leaf_size, - usize block_size, - usize max_key_size, - usize max_edit_size) noexcept; - - /** \brief Returns the passed buffer's memory region, validated as a PackedBlockedLeafPage and - * cast to `const PackedBlockedLeafPage &`. - */ - static const PackedBlockedLeafPage& view_of(const ConstBuffer& buffer) noexcept; - - //+++++++++++-+-+--+----- --- -- - - - - - - big_u64 magic; // +8 -> 8 - little_u32 total_packed_size; // +4 -> 12 - little_u32 blocks_per_art_key; // +4 -> 16 - little_u32 block_size_bytes; // +4 -> 20 - llfs::PackedPointer block0; // +4 -> 24 - - /** \brief Pointer to array that stores, for each block, the starting item index relative to the - * entire leaf. - */ - llfs::PackedPointer> block_starting_item; // +4 -> 28 - - /** \brief Pointer to packed ART index. - */ - llfs::PackedPointer art_block_index; // +4 -> 32 - - //+++++++++++-+-+--+----- --- -- - - - - - - llfs::PageId page_id() const noexcept - { - return (reinterpret_cast(this) - 1)->page_id.unpack(); - } - - Optional page_shard_id_for_block(llfs::PageCache& page_cache, - usize i, - llfs::PageId leaf_page_id) const noexcept - { - const usize block_begin_offset = this->block_page_offset(i); - const usize block_end_offset = block_begin_offset + this->block_size_bytes; - - return page_cache.page_shard_id_for(leaf_page_id, - Interval{block_begin_offset, block_end_offset}); - } - - Optional page_shard_id_for_block(llfs::PageCache& page_cache, - usize i) const noexcept - { - return this->page_shard_id_for_block(page_cache, i, this->page_id()); - } - - usize min_header_shard_size() const noexcept - { - return this->block_page_offset(0); - } - - //----- --- -- - - - - - - usize block_page_offset(usize i) const noexcept - { - return sizeof(llfs::PackedPageHeader) + offsetof(PackedBlockedLeafPage, block0) + - this->block0.offset + i * this->block_size_bytes; - } - - usize block_count() const noexcept - { - return this->block_starting_item->size() - 1; - } - - BlockIterator blocks_begin() const noexcept - { - return BlockIterator{this->block0.get(), (isize)this->block_size_bytes.value()}; - } - - const PackedLeafBlock& blocks_front() const - { - return *this->blocks_begin(); - } - - BlockIterator blocks_end() const noexcept - { - return this->blocks_begin() + this->block_count(); - } - - const PackedLeafBlock& blocks_back() const - { - return *(this->blocks_begin() + (this->block_count() - 1)); - } - - auto blocks() const noexcept - { - return std::ranges::subrange(this->blocks_begin(), this->blocks_end()); - } - - const PackedLeafBlock& block_at(usize block_i) const noexcept - { - return *(this->blocks_begin() + block_i); - } - - BlocksSeq blocks_seq() const noexcept - { - return batt::as_seq(this->blocks()); - } - - Interval item_index_range_of_block(usize i) const noexcept - { - return Interval{ - (*this->block_starting_item)[i].value(), - (*this->block_starting_item)[i + 1].value(), - }; - } - - /** \brief Returns the index of the block that would contain the given key, if it is present in - * this page. - * - * Always returns a valid block index (i.e., less-than this->block_count()) - */ - usize find_block_index_containing_key(const KeyView& key) const noexcept; - - /** \brief Returns a block iterator to the block that would contain the given key, if it is - * present in this page. - * - * Always returns a valid block iterator. - */ - BlockIterator find_block_containing_key(const KeyView& key) const noexcept; - - //----- --- -- - - - - - - /** \brief Returns the number of key/value pairs in this page. - */ - usize item_count() const noexcept - { - return this->block_starting_item->back(); - } - - /** \brief Returns a sequence of all items in the page, in key order. - */ - ItemsSeq items_seq() const noexcept - { - return this->blocks_seq() | batt::seq::map(ItemsSeqFromBlock{}) | batt::seq::flatten(); - } - - /** \brief Returns an item iterator to the first item in the page. - */ - ItemIterator items_begin() const noexcept; - - /** \brief Returns an item iterator one-past the last item in the page. - */ - ItemIterator items_end() const noexcept; - - /** \brief Returns an item iterator to the i-th item in the page. - */ - ItemIterator item_at(usize i) const noexcept; - - /** \brief Returns an iterator to the given key in this page if found or nullptr if not found. - */ - const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept; - - /** \brief Returns an iterator to the first item in this page whose key is not less than `key`; - * if all keys in the page are less than `key`, returns `this->items_end()`. - */ - ItemIterator lower_bound(const KeyView& key) const noexcept; - - //----- --- -- - - - - - - KeyView min_key() const noexcept - { - return this->blocks_front().min_key(); - } - - KeyView max_key() const noexcept - { - return this->blocks_back().max_key(); - } - - SlotSliceSeq slot_slice_seq() const noexcept - { - return this->blocks_seq() | batt::seq::map(SlotSliceFromBlock{}); - } - - template FilterModelT> - ShardedLiveRanges sharded_live_ranges( - const BasicPiecewiseFilter& filter, - const Interval& subrange) const noexcept; -}; - -static_assert(sizeof(PackedBlockedLeafPage) == 32); - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief A view of the header prefix of a PackedBlockedLeafPage. - */ -class PackedBlockedLeafPage::HeaderShardView -{ - public: - using Self = HeaderShardView; - - //+++++++++++-+-+--+----- --- -- - - - - - - static Self view_of(const ConstBuffer& buffer) noexcept - { - const PackedBlockedLeafPage& leaf = PackedBlockedLeafPage::view_of(buffer); - BATT_CHECK_GE(buffer.size(), leaf.min_header_shard_size()); - - return Self{leaf, buffer.size()}; - } - - //+++++++++++-+-+--+----- --- -- - - - - - -#if 0 - Seq load_slices(llfs::PageLoader& loader, - Optional first_key, - Optional last_key, - Optional first_index, - Optional last_index, - const PiecewiseFilter& filter); -#endif - - //+++++++++++-+-+--+----- --- -- - - - - - private: - explicit HeaderShardView(const PackedBlockedLeafPage& leaf, usize header_shard_size) noexcept - : leaf_{&leaf} - , header_shard_size_{header_shard_size} - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - const PackedBlockedLeafPage* leaf_; - usize header_shard_size_; -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp deleted file mode 100644 index 9d415f0..0000000 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.ipp +++ /dev/null @@ -1,316 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_BLOCKED_LEAF_PAGE_IPP - -#include "packed_blocked_leaf_page.hpp" -#include "packed_blocked_leaf_page.item_iterator.hpp" - -#include - -#include -#include - -#include - -namespace turtle_kv { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -StatusOr pack_blocked_leaf_page(const usize block_size, - const ItemRangeT& src_items, - const MutableBuffer& dst_buffer) noexcept -{ - const usize item_count = std::size(src_items); - - //+++++++++++-+-+--+----- --- -- - - - - - // Calculate the number of blocks needed and how many items in each one. - // - SmallVec block_stats; - { - auto src_iter = std::begin(src_items); - const auto src_end = std::end(src_items); - usize blocks_size_remaining = dst_buffer.size() - block_size; - for (;;) { - if (src_iter == src_end) { - break; - } - BATT_CHECK_LT(src_iter, src_end); - - if (blocks_size_remaining < block_size) { - return {batt::StatusCode::kResourceExhausted}; - } - - auto& stats = block_stats.emplace_back( - PackedLeafBlockStats::from(std::ranges::subrange(src_iter, src_end), block_size)); - - blocks_size_remaining -= block_size; - src_iter = std::next(src_iter, stats.item_count); - } - } - const usize block_count = block_stats.size(); - - //+++++++++++-+-+--+----- --- -- - - - - - // Initialize the leaf header. - // - MutableBuffer dst_remaining = dst_buffer; - dst_remaining += sizeof(llfs::PackedPageHeader); - - auto* leaf_header = static_cast(dst_remaining.data()); - dst_remaining += sizeof(PackedBlockedLeafPage); - { - leaf_header->magic = PackedBlockedLeafPage::kMagic; - leaf_header->total_packed_size = 0; - leaf_header->blocks_per_art_key = 0; - leaf_header->block_size_bytes = BATT_CHECKED_CAST(u32, block_size); - leaf_header->block0.offset = 0; - leaf_header->block_starting_item.offset = 0; - leaf_header->art_block_index.offset = 0; - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Pack `block_starting_item` array. - // - { - const usize block_starting_item_array_size = - sizeof(llfs::PackedArray) + sizeof(little_u32) * (block_count + 1); - - auto* block_starting_item = static_cast*>(dst_remaining.data()); - dst_remaining += block_starting_item_array_size; - - block_starting_item->initialize(block_count + 1); - - little_u32* block_start = block_starting_item->data(); - u32 item_i = 0; - for (const PackedLeafBlockStats& stats : block_stats) { - *block_start = item_i; - item_i += stats.item_count; - ++block_start; - } - *block_start = item_count; - - leaf_header->block_starting_item.reset_unsafe(block_starting_item); - } - const llfs::PackedArray& block_starting_item = *(leaf_header->block_starting_item); - - //+++++++++++-+-+--+----- --- -- - - - - - // Calculate blocks_per_art_key based on available space. - // - const usize space_for_art = dst_remaining.size() - block_size * block_count; - SmallVec art_keys; - usize blocks_per_art_key = 1; - //----- --- -- - - - - - const auto items = std::begin(src_items); - const auto key_at = [&items](usize i) { - return get_key(*(items + i)); - }; - //----- --- -- - - - - - for (;;) { - art_keys.clear(); - for (usize block_i = blocks_per_art_key; block_i < block_count; block_i += blocks_per_art_key) { - const usize item_i = block_starting_item[block_i]; - BATT_CHECK_LT(item_i, item_count); - BATT_CHECK_GT(item_i, 0); - - KeyView k0 = key_at(item_i - 1); - KeyView k1 = key_at(item_i); - KeyView common_prefix = llfs::find_common_prefix(0, k0, k1); - KeyView min_k1{k1.data(), common_prefix.size() + 1}; - - art_keys.emplace_back(min_k1); - } - - using artc::packed::PackedARTBuilder; - - batt::StableStringStore string_store; - - BATT_DEBUG_INFO(BATT_INSPECT_RANGE(art_keys) - << BATT_INSPECT(block_count) << BATT_INSPECT(blocks_per_art_key)); - - BATT_ASSIGN_OK_RESULT(auto art_builder, - PackedARTBuilder::from_items(art_keys.begin(), - art_keys.end(), - BATT_OVERLOADS_OF(get_key), - string_store)); - - if (art_builder.get_packed_size() > space_for_art) { - ++blocks_per_art_key; - continue; - } - - MutableBuffer art_buffer{dst_remaining.data(), art_builder.get_packed_size()}; - dst_remaining += art_buffer.size(); - BATT_CHECK_GE(dst_remaining.size(), block_size * block_count); - - BATT_ASSIGN_OK_RESULT(const artc::packed::NodeBase* art_root, art_builder.build(art_buffer)); - - leaf_header->art_block_index.reset_unsafe(art_root); - leaf_header->blocks_per_art_key = BATT_CHECKED_CAST(u32, blocks_per_art_key); - break; - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Shift the remaining buffer forward so it aligns with the nearest block boundary. - // - const usize offset_for_block_align = dst_remaining.size() & (block_size - 1); - dst_remaining += offset_for_block_align; - BATT_CHECK_LE(block_size * block_count, dst_remaining.size()); - - //+++++++++++-+-+--+----- --- -- - - - - - // Pack the blocks. - // - leaf_header->block0.reset_unsafe(static_cast(dst_remaining.data())); - { - auto src_iter = std::begin(src_items); - const auto src_end = std::end(src_items); - usize block_i = 0; - for (const PackedLeafBlockStats& stats : block_stats) { - BATT_DEBUG_INFO(BATT_INSPECT(block_i) << BATT_INSPECT(stats)); - - BATT_CHECK_NE(src_iter, src_end); - auto src_block_items = std::ranges::subrange(src_iter, std::next(src_iter, stats.item_count)); - - BATT_CHECK_GE(dst_remaining.size(), block_size); - MutableBuffer dst_block_buffer{dst_remaining.data(), block_size}; - - auto block_end_iter = - BATT_OK_RESULT_OR_PANIC(pack_leaf_block(src_block_items, dst_block_buffer, stats)); - - BATT_CHECK_EQ(block_end_iter, std::end(src_block_items)); - - src_iter = block_end_iter; - dst_remaining += block_size; - ++block_i; - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Fill in remaining header fields. - // - leaf_header->total_packed_size = BATT_CHECKED_CAST(u32, dst_buffer.size() - dst_remaining.size()); - - //+++++++++++-+-+--+----- --- -- - - - - - // Success! (nothing succeeds like it) - // - return leaf_header; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -/*static*/ const PackedBlockedLeafPage& PackedBlockedLeafPage::view_of( - const ConstBuffer& buffer) noexcept -{ - BATT_CHECK_GT(buffer.size(), sizeof(PackedBlockedLeafPage) + sizeof(llfs::PackedPageHeader)); - - auto* packed = static_cast( - advance_pointer(buffer.data(), sizeof(llfs::PackedPageHeader))); - - BATT_CHECK_EQ(packed->magic, PackedBlockedLeafPage::kMagic); - - return *packed; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::items_begin() const noexcept -{ - auto first_block = this->blocks_begin(); - return ItemIterator{first_block, first_block->items_begin()}; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::items_end() const noexcept -{ - auto last_block = this->blocks_end(); - return ItemIterator{last_block, last_block->items_begin()}; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::item_at(usize i) const noexcept -{ - const llfs::PackedArray& starts = *this->block_starting_item; - - BATT_CHECK_NE(starts.size(), 0); - BATT_CHECK_EQ(starts.front(), 0); - - const auto iter = std::prev(std::upper_bound(starts.begin(), starts.end(), i)); - const isize item_pos_in_block = i - *iter; - const isize block_i = std::distance(starts.begin(), iter); - auto block_iter = this->blocks_begin() + block_i; - - return ItemIterator{block_iter, block_iter->items_begin() + item_pos_in_block}; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline usize PackedBlockedLeafPage::find_block_index_containing_key( - const KeyView& key) const noexcept -{ - using artc::packed::find_lower_bound_rank; - using artc::packed::LowerBoundRank; - - LowerBoundRank result = find_lower_bound_rank(this->art_block_index.get(), key); - - const usize part_i = result.exact ? (result.rank + 1) : result.rank; - const usize block_i = part_i * this->blocks_per_art_key; - - BATT_CHECK_LT(block_i, this->block_count()); - - return block_i; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline PackedLeafBlock::Iterator PackedBlockedLeafPage::find_block_containing_key( - const KeyView& key) const noexcept -{ - return this->blocks_begin() + this->find_block_index_containing_key(key); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline const PackedKeyValueSlotPtr* PackedBlockedLeafPage::find_key( - const KeyView& key) const noexcept -{ - return this->find_block_containing_key(key)->find_key(key); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline PackedBlockedLeafPage::ItemIterator PackedBlockedLeafPage::lower_bound( - const KeyView& key) const noexcept -{ - auto block_iter = this->find_block_containing_key(key); - - const PackedKeyValueSlotPtr* p_slot = block_iter->lower_bound(key); - if (p_slot == block_iter->items_end()) { - ++block_iter; - return ItemIterator{block_iter, block_iter->items_begin()}; - } - - return ItemIterator{block_iter, p_slot}; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template FilterModelT> -inline PackedBlockedLeafPage::ShardedLiveRanges -PackedBlockedLeafPage::sharded_live_ranges(const BasicPiecewiseFilter& filter, - const Interval& subrange) const noexcept -{ - return ShardedLiveRanges{ - this->block_starting_item.get(), - filter.live_subranges_of(subrange), - }; -} - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp deleted file mode 100644 index 664adc6..0000000 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.item_iterator.hpp +++ /dev/null @@ -1,183 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_PACKED_BLOCK_LEAF_PAGE_ITEM_ITERATOR_HPP - -#include "packed_blocked_leaf_page.hpp" - -#include - -namespace turtle_kv { - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -/** \brief Iterator over the items in a blocked leaf page. - */ -class PackedBlockedLeafPage::ItemIterator - : public boost::iterator_facade< // - PackedBlockedLeafPage::ItemIterator, // <- Derived - const PackedKeyValueSlotPtr, // <- Value - std::random_access_iterator_tag, // <- CategoryOrTraversal - const PackedKeyValueSlotPtr&, // <- Reference - isize // <- Difference - > -{ - public: - using Self = ItemIterator; - using iterator_category = std::random_access_iterator_tag; - using value_type = const PackedKeyValueSlotPtr; - using reference = value_type&; - - ItemIterator() = default; - - explicit ItemIterator(BlockIterator block_iter, const PackedKeyValueSlotPtr* slot) noexcept - : block_iter_{block_iter} - , slot_{slot} - { - } - - reference dereference() const - { - return *this->slot_; - } - - bool equal(const Self& other) const - { - return this->block_iter_ == other.block_iter_ && this->slot_ == other.slot_; - } - - void increment() - { - ++this->slot_; - if (this->slot_ == this->block_iter_->items_end()) { - ++this->block_iter_; - this->slot_ = this->block_iter_->items_begin(); - } - } - - void decrement() - { - if (this->slot_ == this->block_iter_->items_begin()) { - --this->block_iter_; - this->slot_ = std::prev(this->block_iter_->items_end()); - } else { - --this->slot_; - } - } - - void advance(isize delta) - { - if (delta == 0) { - return; - } - - isize pos_in_block = this->get_item_pos_in_block(); - - if (delta > 0) { - // Keep stepping through the page one block at a time until we reduce delta to zero. - // - while (delta != 0) { - // Figure out where the current slot is in the current block. - // - const isize remaining_in_block = this->get_remaining_in_block(pos_in_block); - BATT_CHECK_GT(remaining_in_block, 0); - - // If the remaining delta is inside the block, advance the slot pointer and we are done! - // - if (delta < remaining_in_block) { - this->slot_ += delta; - break; - } - // Else reduce delta by the number of slots after this one in the current block. - // - delta -= remaining_in_block; - - // Advance to the next block, resetting the slot pointer. - // - ++this->block_iter_; - this->slot_ = this->block_iter_->items_begin(); - pos_in_block = 0; - } - - } else { // delta < 0 - - delta = -delta; - while (delta != 0) { - BATT_CHECK_GE(pos_in_block, 0); - - // If the remaining delta is inside the block, update the slot pointer and we are done! - // - if (delta <= pos_in_block) { - this->slot_ -= delta; - break; - } - // Else reduce delta by the number of slots before this one in the current block, plus one - // for the current slot. - // - delta -= (pos_in_block + 1); - - // Move to the last item of the previous block. - // - --this->block_iter_; - this->slot_ = std::prev(this->block_iter_->items_end()); - pos_in_block = this->block_iter_->item_count() - 1; - } - } - } - - isize distance_to(const Self& other) const - { - if (this->block_iter_ == other.block_iter_) { - return std::distance(this->slot_, other.slot_); - } - - // Step forward counting items in each block until we reach the same block. - // - if (this->block_iter_ < other.block_iter_) { - isize delta = this->get_remaining_in_block(); - for (auto iter = std::next(this->block_iter_); iter != other.block_iter_; ++iter) { - delta += iter->item_count(); - } - delta += other.get_item_pos_in_block(); - return delta; - } - // Else step backward. - // - isize delta = this->get_item_pos_in_block(); - for (auto iter = std::prev(this->block_iter_); iter != other.block_iter_; --iter) { - delta += iter->item_count(); - } - delta += other.get_remaining_in_block(); - return -delta; - } - - //+++++++++++-+-+--+----- --- -- - - - - - - isize get_item_pos_in_block() const noexcept - { - return std::distance(this->block_iter_->items_begin(), this->slot_); - } - - isize get_remaining_in_block(isize pos_in_block) const noexcept - { - return this->block_iter_->item_count() - pos_in_block; - } - - isize get_remaining_in_block() const noexcept - { - return this->get_remaining_in_block(this->get_item_pos_in_block()); - } - - //+++++++++++-+-+--+----- --- -- - - - - - private: - BlockIterator block_iter_; - const PackedKeyValueSlotPtr* slot_ = nullptr; -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp deleted file mode 100644 index d2a9353..0000000 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.hpp +++ /dev/null @@ -1,56 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_PACKED_BLOCKED_LEAF_PAGE_SHARDED_LIVE_RANGES_HPP - -#include "packed_blocked_leaf_page.hpp" - -#include - -namespace turtle_kv { - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -template FilterModelT> -class PackedBlockedLeafPage::ShardedLiveRanges -{ - public: - using Item = std::pair /*live_item_range*/>; - - //+++++++++++-+-+--+----- --- -- - - - - - - explicit ShardedLiveRanges( - const llfs::PackedArray* block_starts, - BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges) noexcept; - - //+++++++++++-+-+--+----- --- -- - - - - - - Optional peek(); - - Optional next(); - - //+++++++++++-+-+--+----- --- -- - - - - - private: - void advance(); - - usize get_block_count() const noexcept; - - Interval get_block_range(usize block_i) const noexcept; - - void clear_current_range(); - - //+++++++++++-+-+--+----- --- -- - - - - - - const llfs::PackedArray* block_starts_; - usize block_index_; - BasicPiecewiseFilter::LiveSubranges filter_live_ranges_; - Interval current_range_; -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp deleted file mode 100644 index 9192cb2..0000000 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.sharded_live_ranges.ipp +++ /dev/null @@ -1,170 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_PACKED_BLOCKED_LEAF_PAGE_SHARDED_LIVE_RANGES_IPP - -#include "packed_blocked_leaf_page.sharded_live_ranges.hpp" - -namespace turtle_kv { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template FilterModelT> -inline /*explicit*/ PackedBlockedLeafPage::ShardedLiveRanges::ShardedLiveRanges( - const llfs::PackedArray* block_starts, - BasicPiecewiseFilter::LiveSubranges&& filter_live_ranges) noexcept - : block_starts_{block_starts} - , block_index_{0} - , filter_live_ranges_{std::move(filter_live_ranges)} - , current_range_{0, 0} -{ - this->advance(); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template FilterModelT> -inline auto PackedBlockedLeafPage::ShardedLiveRanges::peek() -> Optional -{ - if (this->current_range_.empty()) { - return None; - } - return std::make_pair(this->block_index_, this->current_range_); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template FilterModelT> -inline auto PackedBlockedLeafPage::ShardedLiveRanges::next() -> Optional -{ - Optional item = this->peek(); - if (item) { - this->advance(); - // std::cerr << ".. " << BATT_INSPECT(this->current_range_) << std::endl; - } - return item; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template FilterModelT> -inline void PackedBlockedLeafPage::ShardedLiveRanges::advance() -{ - this->current_range_.lower_bound = this->current_range_.upper_bound; - - Optional> filter_range = this->filter_live_ranges_.peek(); - if (!filter_range) { - return; - } - - // Consume the current range from both current and filter. - // - BATT_CHECK_LE(this->current_range_.upper_bound, filter_range->upper_bound); - filter_range->lower_bound = std::max(filter_range->lower_bound, // - this->current_range_.upper_bound); - - // If the filter range has been consumed, move to the next filter range. - // - if (filter_range->empty()) { - this->filter_live_ranges_.next(); - filter_range = this->filter_live_ranges_.peek(); - - // Once we run out of filter live ranges, we are done. - // - if (!filter_range) { - return; - } - } - - // std::cerr << ".. " << BATT_INSPECT(filter_range) << std::endl; - - const usize block_count = this->get_block_count(); - BATT_CHECK_LT(this->block_index_, block_count); - const usize blocks_remaining = block_count - this->block_index_; - const usize max_probe_steps = BATT_CHECKED_CAST(usize, batt::log2_ceil(blocks_remaining)); - const usize linear_probe_end = this->block_index_ + max_probe_steps; - bool tried_binary_search = false; - - while (this->block_index_ < block_count) { - // Test the intersection of the current block's range with the current filter range; - // if they intersect, then stop here. - // - this->current_range_ = this->get_block_range(this->block_index_) // - .intersection_with(*filter_range); - - if (!this->current_range_.empty()) { - return; - } - - // If we can, continue the linear probe. - // - ++this->block_index_; - if (this->block_index_ <= linear_probe_end) { - continue; - } - - // The binary search fall-back *must* succeed! If we ever find we are about to try it a - // second time, panic. - // - BATT_CHECK(!tried_binary_search); - - // Fall-back to binary search. - // - auto indices = boost::irange(this->block_index_, block_count); - auto iter = - std::lower_bound(indices.begin(), - indices.end(), - *filter_range, - [this](usize i, const Interval& range) { - return Interval::LinearOrder{}(this->get_block_range(i), range); - }); - - // If the first block that might intersect with the filter range is beyond the end of the - // blocks, then we are done. - // - if (iter == indices.end()) { - this->clear_current_range(); - return; - } - - this->block_index_ = *iter; - tried_binary_search = true; - } -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template FilterModelT> -inline usize PackedBlockedLeafPage::ShardedLiveRanges::get_block_count() - const noexcept -{ - return this->block_starts_->size() - 1; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template FilterModelT> -inline Interval PackedBlockedLeafPage::ShardedLiveRanges::get_block_range( - usize block_i) const noexcept -{ - return Interval{ - (*this->block_starts_)[block_i], - (*this->block_starts_)[block_i + 1], - }; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template FilterModelT> -inline void PackedBlockedLeafPage::ShardedLiveRanges::clear_current_range() -{ - this->current_range_.lower_bound = this->current_range_.upper_bound; -} - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp b/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp deleted file mode 100644 index 732154b..0000000 --- a/src/turtle_kv/tree/leaf/packed_blocked_leaf_page.test.cpp +++ /dev/null @@ -1,332 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#include -// -#include - -#include -#include - -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include - -#include -#include -#include - -namespace { - -using namespace batt::int_types; -using namespace batt::constants; - -using batt::MutableBuffer; -using batt::StableStringStore; -using batt::StatusOr; - -using turtle_kv::EditView; -using turtle_kv::Interval; -using turtle_kv::KeyOrder; -using turtle_kv::KeyView; -using turtle_kv::Optional; -using turtle_kv::pack_blocked_leaf_page; -using turtle_kv::PackedBlockedLeafPage; -using turtle_kv::PackedKeyValueSlotPtr; -using turtle_kv::PiecewiseFilter; -using turtle_kv::random_str; -using turtle_kv::ValueView; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// Plan: -// 1. For different random seeds: -// - generate random set of prefixes (~10% of total keys) -// - generate keys using prefixes, with random values -// - sort -// - pack leaf; verify: -// a. all packed keys present and have right values -// b. any unpacked keys at end missing -// c. randomly generated non-present keys not found -// -TEST(TreePackedBlockedLeafPageTest, Random) -{ - const usize kFirstSeed = 0; - const usize kNumSeeds = 250; - const usize kLastSeed = kFirstSeed + kNumSeeds; - const usize kLeafPageSize = 1 * kMiB; - const usize kNumPrefixes = 1000; - const usize kMinPrefixSize = 0; - const usize kMaxPrefixSize = 8; - const usize kMinKeySize = 4; - const usize kMaxKeySize = 48; - const usize kMinValueSize = 0; - const usize kMaxValueSize = 200; - const usize kBlockSize = 8192; - - BATT_CHECK_EQ(batt::bit_count(kLeafPageSize), 1); - - std::uniform_int_distribution pick_pct{0, 99}; - std::geometric_distribution pick_prefix_size{0.5}; - std::uniform_int_distribution pick_prefix{0, kNumPrefixes - 1}; - std::geometric_distribution pick_key_size{0.7}; - std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; - - for (usize seed = kFirstSeed; seed < kLastSeed; ++seed) { - LOG_EVERY_N(INFO, 25) << BATT_INSPECT(seed); - - std::default_random_engine rng{seed}; - - StableStringStore strings; - - //+++++++++++-+-+--+----- --- -- - - - - - // Generate prefixes - // - std::vector prefixes; - { - std::unordered_set used_prefixes; - while (prefixes.size() < kNumPrefixes) { - std::string_view prefix = - random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); - - if (used_prefixes.count(prefix)) { - continue; - } - prefixes.push_back(prefix); - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Generate edits. - // - std::vector edits; - { - usize max_edit_size = 0; - usize max_key_size = 0; - usize total_edits_size = 0; - - std::unordered_set used_keys; - for (;;) { - std::string_view prefix = prefixes[pick_prefix(rng)]; - - std::string_view key = - random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); - - if (used_keys.count(key)) { - continue; - } - used_keys.insert(key); - - std::string_view value = - random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); - - EditView edit{key, ValueView::from_str(value)}; - - const usize edit_size = PackedBlockedLeafPage::packed_edit_size(edit); - - const usize new_max_edit_size = std::max(max_edit_size, edit_size); - const usize new_max_key_size = std::max(max_key_size, key.size()); - - const usize space_available = PackedBlockedLeafPage::estimate_capacity(kLeafPageSize, - kBlockSize, - new_max_key_size, - new_max_edit_size); - - // Stop as soon as adding the next key would exceed the estimated space. - // - if (edit_size + total_edits_size > space_available) { - break; - } - - edits.push_back(edit); - total_edits_size += edit_size; - max_edit_size = new_max_edit_size; - max_key_size = new_max_key_size; - } - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Sort edits by key. - // - std::sort(edits.begin(), edits.end(), KeyOrder{}); - - //+++++++++++-+-+--+----- --- -- - - - - - // Pack a blocked leaf page. - // - using StorageUnit = std::aligned_storage_t<4096, 4096>; - std::vector leaf_storage(kLeafPageSize / sizeof(StorageUnit)); - ASSERT_EQ(sizeof(StorageUnit) * leaf_storage.size(), kLeafPageSize); - - MutableBuffer leaf_buffer{leaf_storage.data(), kLeafPageSize}; - - StatusOr status_or_packed_leaf = - pack_blocked_leaf_page(kBlockSize, edits, leaf_buffer); - - ASSERT_TRUE(status_or_packed_leaf.ok()) << BATT_INSPECT(status_or_packed_leaf.status()); - - const PackedBlockedLeafPage& packed_leaf = PackedBlockedLeafPage::view_of(leaf_buffer); - - ASSERT_EQ(&packed_leaf, *status_or_packed_leaf); - ASSERT_EQ(packed_leaf.min_key(), get_key(edits.front())); - ASSERT_EQ(packed_leaf.max_key(), get_key(edits.back())); - - //+++++++++++-+-+--+----- --- -- - - - - - // Scan over all items in the packed leaf to make sure they are all there. - // - { - PackedBlockedLeafPage::ItemIterator item_iter = packed_leaf.items_begin(); - PackedBlockedLeafPage::ItemIterator items_end = packed_leaf.items_end(); - - std::vector> past_items; - - auto packed_items = packed_leaf.items_seq(); - using Item = decltype(*packed_items.peek()); - Optional prev_key; - Optional prev_item_iter; - - isize position = 0; - - for (const EditView& edit : edits) { - Optional next_packed = packed_items.next(); - - if (prev_key) { - ASSERT_GT(get_key(edit), *prev_key); - } - prev_key = get_key(edit); - - ASSERT_TRUE(next_packed.has_value()); - ASSERT_EQ(get_key(*next_packed), get_key(edit)); - ASSERT_EQ(get_value(*next_packed), get_value(edit)); - - // Test PackedBlockedLeafPage::find_key. - // - const PackedKeyValueSlotPtr* found = packed_leaf.find_key(get_key(edit)); - - ASSERT_NE(found, nullptr); - ASSERT_EQ(found, std::addressof(*item_iter)); - ASSERT_EQ(get_key(*found), get_key(edit)); - ASSERT_EQ(get_value(*found), get_value(edit)) - << BATT_INSPECT_STR(get_key(*found)) << BATT_INSPECT(edit); - - // Test PackedBlockedLeafPage::lower_bound. - // - { - PackedBlockedLeafPage::ItemIterator lb_iter = packed_leaf.lower_bound(get_key(edit)); - - ASSERT_NE(lb_iter, items_end); - ASSERT_EQ(get_key(*lb_iter), get_key(edit)); - ASSERT_EQ(get_value(*lb_iter), get_value(edit)); - } - - ASSERT_EQ(packed_leaf.item_at(position), item_iter); - - ASSERT_NE(item_iter, items_end); - ASSERT_LT(item_iter, items_end); - ASSERT_EQ(std::distance(packed_leaf.items_begin(), item_iter), position); - - if (pick_pct(rng) < 1) { - past_items.push_back(std::make_pair(item_iter, position)); - } - - for (const auto& [past_iter, past_position] : past_items) { - ASSERT_EQ(std::distance(past_iter, item_iter), position - past_position); - ASSERT_EQ(std::distance(item_iter, past_iter), past_position - position); - ASSERT_EQ(past_iter + (position - past_position), item_iter); - ASSERT_EQ(item_iter - (position - past_position), past_iter); - ASSERT_LE(past_iter, item_iter); - ASSERT_GE(item_iter, past_iter) << BATT_INSPECT(position) << BATT_INSPECT(past_position); - } - - if (prev_item_iter) { - ASSERT_EQ(std::next(*prev_item_iter), item_iter); - ASSERT_EQ(*prev_item_iter, std::prev(item_iter)); - } - - prev_item_iter = item_iter; - ++item_iter; - ++position; - } - ASSERT_FALSE(packed_items.peek().has_value()); - } - - //+++++++++++-+-+--+----- --- -- - - - - - // Test ShardedLiveRanges. - // - { - for (usize j = 0; j < 10000; ++j) { - // Drop up to 64 sub-ranges of the leaf. - // - for (usize drop_count = 0; drop_count < 64; ++drop_count) { - PiecewiseFilter leaf_filter; - - std::vector> dropped_ranges; - u32 items_dropped = 0; - const u32 item_count = packed_leaf.item_count(); - - std::tie(items_dropped, dropped_ranges) = - turtle_kv::testing::drop_n_disjoint_intervals_from(&leaf_filter, - drop_count, - Interval{0, item_count}, - rng); - - // Verify the number of expected live items. - // - const u32 expected_live_count = item_count - items_dropped; - - if (drop_count > 1) { - ASSERT_GT(expected_live_count, 0) - << BATT_INSPECT_RANGE(dropped_ranges) << BATT_INSPECT(drop_count) - << BATT_INSPECT(item_count); - } - - u32 actual_live_count = 0; - u32 next_possible_live = 0; - u32 next_possible_block = 0; - - packed_leaf.sharded_live_ranges(leaf_filter, Interval{0, item_count}) | - batt::seq::for_each([&](const std::pair>& live_pair) { - const auto [block_index, live_range] = live_pair; - - BATT_CHECK_GE(block_index, next_possible_block); - BATT_CHECK_LT(block_index, packed_leaf.block_count()); - BATT_CHECK_GE(live_range.lower_bound, next_possible_live) - << BATT_INSPECT(j) << BATT_INSPECT(drop_count) << BATT_INSPECT(live_range) - << BATT_INSPECT(item_count); - BATT_CHECK_LT(live_range.lower_bound, live_range.upper_bound); - BATT_CHECK_LE(live_range.upper_bound, item_count); - - const Interval block_range = - packed_leaf.item_index_range_of_block(block_index); - - BATT_CHECK_GE(live_range.lower_bound, block_range.lower_bound); - BATT_CHECK_LE(live_range.upper_bound, block_range.upper_bound); - - next_possible_live = live_range.upper_bound; - next_possible_block = block_index; - - actual_live_count += live_range.size(); - }); - - ASSERT_EQ(actual_live_count, expected_live_count) - << BATT_INSPECT(j) << BATT_INSPECT(drop_count); - } - } - } - } -} - -} // namespace diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block.hpp deleted file mode 100644 index 997f0ef..0000000 --- a/src/turtle_kv/tree/leaf/packed_leaf_block.hpp +++ /dev/null @@ -1,170 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_HPP - -#include "packed_leaf_block_stats.hpp" - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include - -namespace turtle_kv { - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -struct PackedLeafBlock { - static constexpr u32 kMagic = 0x7370b49full; - - //+++++++++++-+-+--+----- --- -- - - - - - - class Iterator; - - using BlockItemsSeq = batt::SubRangeSeq>; - - //+++++++++++-+-+--+----- --- -- - - - - - - big_u32 magic; // +4 = 4 - little_u16 shared_prefix_size; // +2 = 6 - PackedKeyValueSlotPtr items_[1]; // +2 = 8 - - //+++++++++++-+-+--+----- --- -- - - - - - - template - static usize packed_edit_size(const EditT& edit) noexcept - { - const usize slot_size = packed_key_value_slot_size(edit); - const usize edit_size = slot_size + sizeof(PackedKeyValueSlotPtr); - - return edit_size; - } - - static constexpr usize capacity(usize block_size) noexcept - { - return block_size - std::min(block_size, sizeof(PackedLeafBlock)); - } - - /** \brief Returns the passed buffer's memory region, validated as a PackedLeafBlock and cast to - * `const PackedLeafBlock &`. - */ - static const PackedLeafBlock& view_of(const ConstBuffer& buffer) noexcept; - - //+++++++++++-+-+--+----- --- -- - - - - - - usize item_count() const noexcept - { - return this->items_end() - this->items_begin(); - } - - KeyView key_at(usize i) const noexcept - { - return this->items_[i]->key_view(); - } - - ValueView value_at(usize i) const noexcept - { - return this->items_[i]->value_view(&this->items_[i]); - } - - EditView edit_at(usize i) const noexcept - { - auto& packed = *this->items_[i]; - return EditView{packed.key_view(), packed.value_view(&this->items_[i])}; - } - - Optional item_at(usize i) const noexcept - { - return to_item_view(this->edit_at(i)); - } - - const PackedKeyValueSlotPtr& front_item() const noexcept - { - return this->items_[0]; - } - - const PackedKeyValueSlotPtr& back_item() const noexcept - { - return this->items_[this->item_count() - 1]; - } - - BlockItemsSeq items_seq() const noexcept - { - return batt::as_seq(this->items_slice()); - } - - const PackedKeyValueSlotPtr* items_begin() const noexcept - { - return this->items_; - } - - const PackedKeyValueSlotPtr* items_end() const noexcept - { - return ((const PackedKeyValueSlotPtr*)this->items_[0].get()) - 1; - } - - Slice items_slice() const noexcept - { - return as_slice(this->items_begin(), this->items_end()); - } - - Slice items_slice(Optional key_lower_bound, - Optional key_upper_bound) const noexcept; - - KeyView min_key() const noexcept - { - return get_key(this->front_item()); - } - - KeyView max_key() const noexcept - { - return get_key(this->back_item()); - } - - KeyView shared_key_prefix() const noexcept - { - return this->min_key().substr(0, this->shared_prefix_size); - } - - /** \brief Returns an iterator to the given key in this block if found or nullptr if not found. - */ - const PackedKeyValueSlotPtr* find_key(const KeyView& key) const noexcept; - - /** \brief Returns an iterator to the first item in this block whose key is not less than `key`; - * if all keys in the block are less than `key`, returns `this->items_end()`. - */ - const PackedKeyValueSlotPtr* lower_bound(const KeyView& key) const noexcept; -}; - -static_assert(sizeof(PackedLeafBlock) == 8); - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template ()))>> -StatusOr pack_leaf_block(const RangeT& src, - MutableBuffer dst, - const Optional& stats = None) noexcept; - -} // namespace turtle_kv - -#include "packed_leaf_block.ipp" diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.ipp b/src/turtle_kv/tree/leaf/packed_leaf_block.ipp deleted file mode 100644 index 25d842c..0000000 --- a/src/turtle_kv/tree/leaf/packed_leaf_block.ipp +++ /dev/null @@ -1,162 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_IPP - -#include "packed_leaf_block.hpp" -#include "packed_leaf_block_stats.ipp" - -#include - -#include -#include - -#include - -#include -#include - -#include - -namespace turtle_kv { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline StatusOr pack_leaf_block(const RangeT& src, - MutableBuffer dst, - const Optional& opt_stats) noexcept -{ - if (dst.size() < sizeof(PackedLeafBlock)) { - return {batt::StatusCode::kResourceExhausted}; - } - - PackedLeafBlockStats stats = opt_stats.or_else([&] { - return PackedLeafBlockStats::from(src, dst.size()); - }); - - if (stats.block_size != dst.size()) { - return {batt::StatusCode::kResourceExhausted}; - } - - PackedLeafBlock* block = static_cast(dst.data()); - { - block->magic = PackedLeafBlock::kMagic; - block->items_[0].offset = BATT_CHECKED_CAST( - u32, - byte_distance(block->items_, advance_pointer(&block->items_[1], stats.item_ptr_bytes))); - } - - //----- --- -- - - - - - // Pack all slot data. - // - PackedKeyValueSlotPtr* pp_slot = block->items_; - void* p_slot = const_cast(pp_slot->get()); - void* const dst_end = advance_pointer(dst.data(), dst.size()); - - IterT src_iter = std::begin(src); - const IterT src_end = std::next(src_iter, stats.item_count); - for (; src_iter != src_end; ++src_iter) { - const usize slot_size = pack_key_value_slot(*src_iter, p_slot); - p_slot = advance_pointer(p_slot, slot_size); - BATT_CHECK_LE(p_slot, dst_end); - ++pp_slot; - pp_slot->offset = byte_distance(pp_slot, p_slot); - - BATT_CHECK_EQ((void*)pp_slot->get(), (void*)p_slot); - } - - //----- --- -- - - - - - // Set the common prefix. - // - block->shared_prefix_size = - BATT_CHECKED_CAST(u16, - llfs::find_common_prefix(0, block->min_key(), block->max_key()).size()); - - return {src_iter}; -} - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// struct PackedLeafBlock - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline /*static*/ const PackedLeafBlock& PackedLeafBlock::view_of( - const ConstBuffer& buffer) noexcept -{ - BATT_CHECK_GE(buffer.size(), sizeof(PackedLeafBlock)); - - const auto* block = static_cast(buffer.data()); - - BATT_CHECK_EQ(block->magic, PackedLeafBlock::kMagic); - - return *block; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline const PackedKeyValueSlotPtr* PackedLeafBlock::find_key(const KeyView& key) const noexcept -{ - const auto convert_result = [](auto&& first_last_pair) -> const PackedKeyValueSlotPtr* { - if (first_last_pair.first == first_last_pair.second) { - return nullptr; - } - return first_last_pair.first; - }; - - if (this->shared_prefix_size > 0) { - const usize prefix_size = this->shared_prefix_size; - if (key.size() < prefix_size) { - return nullptr; - } - auto order = batt::compare(key.substr(0, prefix_size), this->shared_key_prefix()); - if (order != batt::Order::Equal) { - return nullptr; - } - - return convert_result( - std::equal_range(this->items_begin(), this->items_end(), key, KeySuffixOrder{prefix_size})); - } - - return convert_result(std::equal_range(this->items_begin(), - this->items_end(), - key, - [](const auto& l, const auto& r) { - return batt::compare(get_key(l), get_key(r)) == - batt::Order::Less; - })); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline const PackedKeyValueSlotPtr* PackedLeafBlock::lower_bound(const KeyView& key) const noexcept -{ - return std::lower_bound(this->items_begin(), - this->items_end(), - key, - [](const auto& l, const auto& r) { - return batt::compare(get_key(l), get_key(r)) == batt::Order::Less; - }); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -inline Slice PackedLeafBlock::items_slice( - Optional key_lower_bound, - Optional key_upper_bound) const noexcept -{ - auto [first, last] = std::equal_range(this->items_begin(), - this->items_end(), - Interval{key_lower_bound.or_else(global_min_key), - key_upper_bound.or_else(global_max_key)}, - ExtendedKeyRangeOrder{}); - return as_slice(first, last); -} - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp deleted file mode 100644 index ade706d..0000000 --- a/src/turtle_kv/tree/leaf/packed_leaf_block.iterator.hpp +++ /dev/null @@ -1,104 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_ITERATOR_HPP - -#include "packed_leaf_block.hpp" - -#include -#include - -#include - -namespace turtle_kv { - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -class PackedLeafBlock::Iterator - : public boost::iterator_facade< // - PackedLeafBlock::Iterator, // <- Derived - const PackedLeafBlock, // <- Value - std::random_access_iterator_tag, // <- CategoryOrTraversal - const PackedLeafBlock&, // <- Reference - isize // <- Difference - > -{ - public: - using Self = Iterator; - using iterator_category = std::random_access_iterator_tag; - using value_type = const PackedLeafBlock; - using reference = const PackedLeafBlock&; - - //+++++++++++-+-+--+----- --- -- - - - -- - - Iterator() = default; - - explicit Iterator(const PackedLeafBlock* block, isize block_size) noexcept - : block_{block} - , block_size_{block_size} - { - } - - //+++++++++++-+-+--+----- --- -- - - - -- - - reference dereference() const - { - return *this->block_; - } - - bool equal(const Self& other) const - { - return this->block_ == other.block_ && this->block_size_ == other.block_size_; - } - - void increment() - { - this->advance(1); - } - - void decrement() - { - this->advance(-1); - } - - void advance(isize delta) - { - this->block_ = static_cast( - advance_pointer(this->block_, delta * this->block_size_)); - } - - isize distance_to(const Self& other) const - { - return (byte_distance(this->block_, other.block_)) / this->block_size_; - } - - //+++++++++++-+-+--+----- --- -- - - - -- - - const PackedLeafBlock* block() const noexcept - { - return this->block_; - } - - usize block_size() const noexcept - { - return static_cast(this->block_size_); - } - - isize block_isize() const noexcept - { - return this->block_size_; - } - - //+++++++++++-+-+--+----- --- -- - - - -- - private: - const PackedLeafBlock* block_ = nullptr; - isize block_size_ = 0; -}; - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp b/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp deleted file mode 100644 index f19fefc..0000000 --- a/src/turtle_kv/tree/leaf/packed_leaf_block.test.cpp +++ /dev/null @@ -1,173 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#include -// -#include - -#include -#include - -#include - -#include - -#include -#include -#include - -namespace { - -using namespace batt::int_types; - -using batt::MutableBuffer; -using batt::StableStringStore; -using batt::StatusOr; - -using turtle_kv::EditView; -using turtle_kv::KeyOrder; -using turtle_kv::KeyView; -using turtle_kv::random_str; -using turtle_kv::ValueView; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -TEST(TreePackedLeafBlockTest, Random) -{ - const usize kNumSeeds = 1000; - const usize kNumNotFoundQueries = 100; - const usize kNumLowerBoundQueries = 500; - const usize kMinPrefixSize = 0; - const usize kMaxPrefixSize = 8; - const usize kMinKeySize = 4; - const usize kMaxKeySize = 48; - const usize kMinValueSize = 0; - const usize kMaxValueSize = 200; - const usize kBlockSize = 8192; - - std::geometric_distribution pick_prefix_size{0.5}; - std::geometric_distribution pick_key_size{0.7}; - std::uniform_int_distribution pick_value_size{0, kMaxValueSize - kMinValueSize}; - - for (usize seed = 0; seed < kNumSeeds; ++seed) { - std::default_random_engine rng{seed}; - - StableStringStore strings; - - std::string_view prefix = - random_str(rng, pick_prefix_size, kMinPrefixSize, kMaxPrefixSize, strings); - - std::unordered_set used_keys; - std::vector src_edits; - - // Generate enough random edits to fill a block. - // - usize src_size = 0; - while (src_size < kBlockSize) { - std::string_view key = - random_str(rng, pick_key_size, kMinKeySize, kMaxKeySize, strings, prefix); - if (used_keys.count(key)) { - continue; - } - used_keys.insert(key); - - std::string_view value = - random_str(rng, pick_value_size, kMinValueSize, kMaxValueSize, strings); - - src_edits.push_back(EditView{key, ValueView::from_str(value)}); - src_size += key.size() + value.size(); - } - std::sort(src_edits.begin(), src_edits.end(), KeyOrder{}); - - // Pack a block. - // - std::array block_buffer; - block_buffer.fill('!'); - auto dst_buffer = MutableBuffer{block_buffer.data(), kBlockSize}; - - StatusOr::const_iterator> consumed_src_end = - turtle_kv::pack_leaf_block(src_edits, dst_buffer); - - ASSERT_TRUE(consumed_src_end.ok()); - - for (usize i = kBlockSize; i < kBlockSize * 2; ++i) { - ASSERT_EQ(block_buffer[i], '!') << BATT_INSPECT(i); - } - - const usize packed_count = *consumed_src_end - src_edits.begin(); - const auto& packed_block = turtle_kv::PackedLeafBlock::view_of(dst_buffer); - - ASSERT_EQ(packed_block.shared_prefix_size.value(), prefix.size()); - - usize found_count = 0; - for (const EditView& src_edit : src_edits) { - auto* found_ptr = packed_block.find_key(get_key(src_edit)); - if (found_count < packed_count) { - ASSERT_NE(found_ptr, nullptr) - << BATT_INSPECT(src_edit) << BATT_INSPECT(found_count) << BATT_INSPECT(packed_count); - ++found_count; - - ASSERT_EQ(get_key(*found_ptr), get_key(src_edit)); - ASSERT_EQ(get_value(*found_ptr), get_value(src_edit)); - } else { - ASSERT_EQ(found_ptr, nullptr); - } - } - - // Run empty queries. - // - for (usize i = 0; i < kNumNotFoundQueries; ++i) { - std::string_view key; - for (;;) { - key = random_str(rng, - pick_key_size, - kMinKeySize + prefix.size(), - kMaxKeySize + prefix.size(), - strings); - if (!used_keys.count(key)) { - break; - } - } - - ASSERT_EQ(packed_block.find_key(key), nullptr); - } - - // Run lower bound queries. - // - for (usize i = 0; i < kNumLowerBoundQueries; ++i) { - std::string_view key = (i % 2) ? random_str(rng, - pick_key_size, - kMinKeySize + prefix.size(), - kMaxKeySize + prefix.size(), - strings) - : random_str(rng, // - pick_key_size, - kMinKeySize, - kMaxKeySize, - strings, - prefix); - - const auto expected_iter = - std::lower_bound(src_edits.begin(), src_edits.end(), key, KeyOrder{}); - - const usize expected_i = std::distance(src_edits.begin(), expected_iter); - - const auto actual_iter = packed_block.lower_bound(key); - - const usize actual_i = std::distance(packed_block.items_begin(), actual_iter); - - if (expected_i >= packed_count) { - ASSERT_EQ(actual_i, packed_count); - } else { - ASSERT_EQ(actual_i, expected_i); - } - } - } -} - -} // namespace diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp deleted file mode 100644 index 2d8360f..0000000 --- a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.hpp +++ /dev/null @@ -1,38 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_STATS_HPP - -#include - -#include - -#include - -namespace turtle_kv { - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -struct PackedLeafBlockStats { - usize block_size; - usize item_count; - usize item_slot_bytes; - usize item_ptr_bytes; - - //+++++++++++-+-+--+----- --- -- - - - - - - template - static PackedLeafBlockStats from(const RangeT& src, usize block_size) noexcept; -}; - -BATT_OBJECT_PRINT_IMPL((inline), - PackedLeafBlockStats, - (block_size, item_count, item_slot_bytes, item_ptr_bytes)) - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp b/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp deleted file mode 100644 index a9b9bf7..0000000 --- a/src/turtle_kv/tree/leaf/packed_leaf_block_stats.ipp +++ /dev/null @@ -1,56 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_LEAF_PACKED_LEAF_BLOCK_STATS_IPP - -namespace turtle_kv { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -inline /*static*/ PackedLeafBlockStats PackedLeafBlockStats::from(const RangeT& src, - usize dst_size) noexcept -{ - PackedLeafBlockStats stats{ - .block_size = 0, - .item_count = 0, - .item_slot_bytes = 0, - .item_ptr_bytes = 0, - }; - - if (dst_size < sizeof(PackedLeafBlock)) { - return stats; - } - stats.block_size = dst_size; - usize offset = 0; - dst_size -= sizeof(PackedLeafBlock); - offset += sizeof(PackedLeafBlock); - - for (const auto& src_item : src) { - const usize slot_size = packed_key_value_slot_size(src_item); - const usize total_item_size = slot_size + sizeof(PackedKeyValueSlotPtr); - if (dst_size < total_item_size) { - break; - } - stats.item_count += 1; - stats.item_slot_bytes += slot_size; - stats.item_ptr_bytes += sizeof(PackedKeyValueSlotPtr); - dst_size -= total_item_size; - offset += total_item_size; - - if constexpr (false) { - LOG(INFO) << BATT_INSPECT(offset) << BATT_INSPECT_STR(get_key(src_item)) - << BATT_INSPECT(stats.item_count); - } - } - - return stats; -} - -} // namespace turtle_kv diff --git a/src/turtle_kv/tree/packed_leaf_block_scanner.hpp b/src/turtle_kv/tree/packed_leaf_block_scanner.hpp deleted file mode 100644 index 5f5d81a..0000000 --- a/src/turtle_kv/tree/packed_leaf_block_scanner.hpp +++ /dev/null @@ -1,75 +0,0 @@ -//=##=##=#==#=#==#===#+==#+==========+==+=+=+=+=+=++=+++=+++++=-++++=-+++++++++++ -// -// Part of the TurtleKV Project, under Apache License v2.0. -// See https://www.apache.org/licenses/LICENSE-2.0 for license information. -// SPDX short identifier: Apache-2.0 -// -//+++++++++++-+-+--+----- --- -- - - - - - -#pragma once -#define TURTLE_KV_TREE_PACKED_LEAF_BLOCK_SCANNER_HPP - -#include - -#include - -#include - -#include - -#include - -namespace turtle_kv { - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -template -concept PackedLeafBlockProvider = requires(T& provider, usize block_index) { - { provider.get_block(block_index) } -> std::convertible_to>; -}; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -template -class PackedLeafBlockScanner -{ - public: - class Impl - { - public: - using Item = StatusOr; - - Optional poll() noexcept - { - } - - Optional next() noexcept - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - private: - void advance() noexcept - { - } - - //+++++++++++-+-+--+----- --- -- - - - - - - PackedBlockedLeafPage::HeaderShardView header_; - - BlockProviderT& provider_; - - PiecewiseFilter& filter_; - - usize block_index_; - - Optional> block_; - }; - - //+++++++++++-+-+--+----- --- -- - - - - - - private: - Impl* impl_; -}; - -} // namespace turtle_kv From 97b2d12604b29dfc421a009eab96b50c21667701 Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Tue, 18 Aug 2026 18:16:01 -0400 Subject: [PATCH 12/13] Remove unrelated changes. --- src/turtle_kv/core/key_range.hpp | 25 ++- src/turtle_kv/core/key_view.hpp | 11 +- src/turtle_kv/core/packed_key_value_slot.hpp | 196 +------------------ src/turtle_kv/core/value_view.hpp | 12 +- 4 files changed, 23 insertions(+), 221 deletions(-) diff --git a/src/turtle_kv/core/key_range.hpp b/src/turtle_kv/core/key_range.hpp index 2e1c198..fc5d57d 100644 --- a/src/turtle_kv/core/key_range.hpp +++ b/src/turtle_kv/core/key_range.hpp @@ -34,12 +34,27 @@ inline CInterval get_key_range(const Chunk& chunk) }; } -template -inline CInterval get_key_range(const T& has_key_view) +inline CInterval get_key_range(const EditView& edit) { return CInterval{ - .lower_bound = get_key(has_key_view), - .upper_bound = get_key(has_key_view), + .lower_bound = get_key(edit), + .upper_bound = get_key(edit), + }; +} + +inline CInterval get_key_range(const ItemView& item) +{ + return CInterval{ + .lower_bound = get_key(item), + .upper_bound = get_key(item), + }; +} + +inline CInterval get_key_range(const KeyView& key) +{ + return CInterval{ + .lower_bound = key, + .upper_bound = key, }; } @@ -74,4 +89,4 @@ struct ExtendedKeyRangeOrder : llfs::KeyRangeOrder { } }; -} // namespace turtle_kv +} // namespace turtle_kv \ No newline at end of file diff --git a/src/turtle_kv/core/key_view.hpp b/src/turtle_kv/core/key_view.hpp index bbf16b6..bf43e7e 100644 --- a/src/turtle_kv/core/key_view.hpp +++ b/src/turtle_kv/core/key_view.hpp @@ -85,13 +85,4 @@ inline usize packed_key_data_size(const KeyView& key) return key.size(); } -template -concept HasKeyView = requires(const T& obj) { - { get_key(obj) } -> std::convertible_to; -}; - -static_assert(HasKeyView); -static_assert(HasKeyView); -static_assert(HasKeyView); - -} // namespace turtle_kv +} // namespace turtle_kv \ No newline at end of file diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index ce42137..3440f98 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -17,156 +17,8 @@ #include #include -#include - namespace turtle_kv { -struct PackedKeyValueSlot; - -using PackedKeyValueSlotPtr = llfs::PackedPointer; - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// -struct PackedKeyValueSlot { - little_u16 key_size; - char key_data_[0]; - - //----- --- -- - - - - - // u8 key_bytes[this->key_size] - //----- --- -- - - - - - // u8 op_code - // u8 value_bytes[this->item_size - offsetof(this->value_bytes)] - //----- --- -- - - - - - - //+++++++++++-+-+--+----- --- -- - - - - - - PackedKeyValueSlot(const PackedKeyValueSlot&) = delete; - PackedKeyValueSlot& operator=(const PackedKeyValueSlot&) = delete; - - //+++++++++++-+-+--+----- --- -- - - - - - - usize slot_size(const PackedKeyValueSlotPtr* p_this) const noexcept - { - const PackedKeyValueSlotPtr* const p_next = p_this + 1; - return usize{p_next->offset.value()} - usize{p_this->offset.value()} + - sizeof(PackedKeyValueSlotPtr); - } - - const char* key_data() const noexcept - { - return this->key_data_; - } - - KeyView key_view() const noexcept - { - return KeyView{this->key_data_, this->key_size}; - } - - const char* value_data() const noexcept - { - return this->key_data() + (this->key_size + 1); - } - - const char* value_data_end(const PackedKeyValueSlotPtr* p_this) const noexcept - { - return this->value_data_end(/*size_of_slot=*/this->slot_size(p_this)); - } - - const char* value_data_end(usize size_of_slot) const noexcept - { - return reinterpret_cast(this) + size_of_slot; - } - - usize value_size(const PackedKeyValueSlotPtr* p_this) const noexcept - { - return this->value_size(/*size_of_slot=*/this->slot_size(p_this)); - } - - usize value_size(usize size_of_slot) const noexcept - { - return this->value_data_end(size_of_slot) - this->value_data(); - } - - ValueView::OpCode value_op_code() const noexcept - { - return static_cast(this->key_data_[this->key_size]); - } - - ValueView value_view(const PackedKeyValueSlotPtr* p_this) const noexcept - { - return this->value_view(/*size_of_slot=*/this->slot_size(p_this)); - } - - ValueView value_view(usize size_of_slot) const noexcept - { - return ValueView::from_packed( - this->value_op_code(), - std::string_view{this->value_data(), this->value_size(size_of_slot)}); - } -}; - -inline KeyView get_key(const PackedKeyValueSlot& packed_slot) noexcept -{ - return packed_slot.key_view(); -} - -//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+--------------- -// TODO [tastolfi 2026-05-31] use 16-bit pointer tagging to store slot_size with the pointer in one -// 64-bit word. -// -struct PackedKeyValueSlotRef { - const PackedKeyValueSlot* p_slot; - usize slot_size; -}; - -inline KeyView get_key(const PackedKeyValueSlotRef& slot_ref) noexcept -{ - return slot_ref.p_slot->key_view(); -} - -inline ValueView get_value(const PackedKeyValueSlotRef& slot_ref) noexcept -{ - return slot_ref.p_slot->value_view(slot_ref.slot_size); -} - -inline const PackedKeyValueSlotRef& to_key_value_slot_ref(const PackedKeyValueSlotRef& ref) noexcept -{ - return ref; -} - -inline PackedKeyValueSlotRef to_key_value_slot_ref(const PackedKeyValueSlotPtr* pp_slot) noexcept -{ - return PackedKeyValueSlotRef{ - .p_slot = pp_slot->get(), - .slot_size = pp_slot->get()->slot_size(pp_slot), - }; -} - -inline PackedKeyValueSlotRef to_key_value_slot_ref(const ConstBuffer& slot_buffer) noexcept -{ - return PackedKeyValueSlotRef{ - .p_slot = static_cast(slot_buffer.data()), - .slot_size = slot_buffer.size(), - }; -} - -inline KeyView get_key(const PackedKeyValueSlotPtr& p_kv) noexcept -{ - return get_key(*p_kv); -} - -inline ValueView get_value(const PackedKeyValueSlotPtr& p_kv) noexcept -{ - return p_kv->value_view(std::addressof(p_kv)); -} - -template -concept ConvertibleToKeyValueSlotRef = requires(const T& obj) { - { to_key_value_slot_ref(obj) } -> std::convertible_to; -}; - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// /** \brief Returns the size required (in bytes) to pack a slot with the passed key and * value. */ @@ -178,21 +30,6 @@ inline usize packed_key_value_slot_size(const KeyView& key, const ValueView& val + value.size(); } -template -inline usize packed_key_value_slot_size(const T& obj) noexcept -{ - return to_key_value_slot_ref(obj).slot_size; -} - -template - requires HasKeyView && HasValueView -inline usize packed_key_value_slot_size(const T& obj) noexcept -{ - return packed_key_value_slot_size(get_key(obj), get_value(obj)); -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// /** \brief Serializes the passed key and value into the destination buffer. */ inline std::pair pack_key_value_slot(const KeyView& key, @@ -235,37 +72,6 @@ inline std::pair pack_key_value_slot(const KeyView& key, })); } -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -/** \brief - */ -template - requires HasKeyView && HasValueView -inline usize pack_key_value_slot(const T& src, void* dst) noexcept -{ - const KeyView& key = get_key(src); - const ValueView& value = get_value(src); - const usize slot_size = packed_key_value_slot_size(key, value); - - pack_key_value_slot(key, value, MutableBuffer{dst, slot_size}); - - return slot_size; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// -/** \brief - */ -template -inline usize pack_key_value_slot(const T& src, void* dst) noexcept -{ - const PackedKeyValueSlotRef& slot_ref = to_key_value_slot_ref(src); - std::memcpy(dst, slot_ref.p_slot, slot_ref.slot_size); - return slot_ref.slot_size; -} - -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// /** \brief Unpacks a key/value pair from the passed packed slot buffer. */ inline StatusOr> unpack_key_value_slot(ConstBuffer payload) @@ -285,4 +91,4 @@ inline StatusOr> unpack_key_value_slot(ConstBuffer return std::make_pair(KeyView{key_data, *p_key_len}, value); } -} // namespace turtle_kv +} // namespace turtle_kv \ No newline at end of file diff --git a/src/turtle_kv/core/value_view.hpp b/src/turtle_kv/core/value_view.hpp index 3fb4d7b..97448e1 100644 --- a/src/turtle_kv/core/value_view.hpp +++ b/src/turtle_kv/core/value_view.hpp @@ -404,14 +404,4 @@ inline bool decays_to_item(const ValueView& value) return false; } -//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - - -// - -template -concept HasValueView = requires(const T& obj) { - { get_value(obj) } -> std::convertible_to; -}; - -static_assert(HasValueView); - -} // namespace turtle_kv +} // namespace turtle_kv \ No newline at end of file From 7a6cd448df5cfd1580d84274b92a4ce7b9d65275 Mon Sep 17 00:00:00 2001 From: Vidya Silai Date: Tue, 18 Aug 2026 18:18:28 -0400 Subject: [PATCH 13/13] Fix new lines. --- src/turtle_kv/core/key_range.hpp | 2 +- src/turtle_kv/core/key_view.hpp | 2 +- src/turtle_kv/core/packed_key_value_slot.hpp | 2 +- src/turtle_kv/core/value_view.hpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/turtle_kv/core/key_range.hpp b/src/turtle_kv/core/key_range.hpp index fc5d57d..16febe1 100644 --- a/src/turtle_kv/core/key_range.hpp +++ b/src/turtle_kv/core/key_range.hpp @@ -89,4 +89,4 @@ struct ExtendedKeyRangeOrder : llfs::KeyRangeOrder { } }; -} // namespace turtle_kv \ No newline at end of file +} // namespace turtle_kv diff --git a/src/turtle_kv/core/key_view.hpp b/src/turtle_kv/core/key_view.hpp index bf43e7e..7c91fa8 100644 --- a/src/turtle_kv/core/key_view.hpp +++ b/src/turtle_kv/core/key_view.hpp @@ -85,4 +85,4 @@ inline usize packed_key_data_size(const KeyView& key) return key.size(); } -} // namespace turtle_kv \ No newline at end of file +} // namespace turtle_kv diff --git a/src/turtle_kv/core/packed_key_value_slot.hpp b/src/turtle_kv/core/packed_key_value_slot.hpp index 3440f98..9bc08d2 100644 --- a/src/turtle_kv/core/packed_key_value_slot.hpp +++ b/src/turtle_kv/core/packed_key_value_slot.hpp @@ -91,4 +91,4 @@ inline StatusOr> unpack_key_value_slot(ConstBuffer return std::make_pair(KeyView{key_data, *p_key_len}, value); } -} // namespace turtle_kv \ No newline at end of file +} // namespace turtle_kv diff --git a/src/turtle_kv/core/value_view.hpp b/src/turtle_kv/core/value_view.hpp index 97448e1..e79673f 100644 --- a/src/turtle_kv/core/value_view.hpp +++ b/src/turtle_kv/core/value_view.hpp @@ -404,4 +404,4 @@ inline bool decays_to_item(const ValueView& value) return false; } -} // namespace turtle_kv \ No newline at end of file +} // namespace turtle_kv