diff --git a/.github/copilot-setup-steps.yml b/.github/copilot-setup-steps.yml deleted file mode 100644 index cc444ea..0000000 --- a/.github/copilot-setup-steps.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Copilot Setup Steps - -on: workflow_dispatch - -jobs: - setup: - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v4 - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y ninja-build liburing-dev clang-19 clang-tools-19 unzip nodejs npm - - - name: Download lexy - run: | - curl -fLo /tmp/lexy.zip https://lexy.foonathan.net/download/lexy-src.zip - unzip -q /tmp/lexy.zip -d /tmp/lexy_extracted - - - name: Configure CMake - run: cmake -B build -G Ninja -DBUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_C_COMPILER=clang-19 -DFETCHCONTENT_SOURCE_DIR_LEXY=/tmp/lexy_extracted - - - name: Build - run: ninja -C build - - - name: Install test_cql.js dependencies - run: npm install - working-directory: extra/node diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 0000000..a304906 --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,35 @@ +name: Copilot Setup Steps + +on: + workflow_dispatch: + push: + paths: + - .github/workflows/copilot-setup-steps.yml + pull_request: + paths: + - .github/workflows/copilot-setup-steps.yml + +jobs: + copilot-setup-steps: + runs-on: ubuntu-latest + + permissions: + contents: read + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build liburing-dev clang-19 clang-tools-19 nodejs npm + + - name: Configure CMake + run: cmake -B build -G Ninja -DBUILD_TESTS=ON -DPLEXDB_LOG_ENABLED=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_C_COMPILER=clang-19 + + - name: Build + run: ninja -C build + + - name: Install test_cql.js dependencies + run: npm install + working-directory: extra/node diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fbadf85..0a7d8f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,7 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=clang++-19 \ -DCMAKE_C_COMPILER=clang-19 \ + -DPLEXDB_DEBUG=OFF \ -DBUILD_TESTS=OFF \ -DBUILD_SHARED_LIBS=OFF diff --git a/CMakeLists.txt b/CMakeLists.txt index 10a8d13..e493ba6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,12 @@ if(DISABLE_EXCEPTIONS) add_compile_options("$<$:-fno-exceptions>" "$<$:-fno-rtti>") endif() endif() +if(PLEXDB_DEBUG) + add_compile_definitions(PLEXDB_DEBUG=1) +endif() +if(PLEXDB_LOG_ENABLED) + add_compile_definitions(PLEXDB_LOG_ENABLED=1) +endif() # ============================================================================= # Build diff --git a/macros/macros.h b/macros/macros.h index 2fb924b..e947f5c 100644 --- a/macros/macros.h +++ b/macros/macros.h @@ -126,7 +126,7 @@ // utilities -#if !defined(PLEXDB_DEBUG) +#if defined(PLEXDB_DEBUG) #define PLEXDB_DEBUG 1 #define PLEXDB_DEBUG_X(x) x #else diff --git a/objstore/CMakeLists.txt b/objstore/CMakeLists.txt index 1e4b093..7a495dc 100644 --- a/objstore/CMakeLists.txt +++ b/objstore/CMakeLists.txt @@ -15,14 +15,19 @@ if(OBJSTORE_IS_MAIN_PROJECT) option(BUILD_TESTS "Build unit tests" ON) option(OBJSTORE_BUILD_SERVER "Build objstore server" ON) option(PLEXDB_LOG_ENABLED "Enable structured logging hooks" OFF) + option(PLEXDB_DEBUG "Build debug checks" ON) add_subdirectory(../plexdb plexdb) -endif() -if(PLEXDB_LOG_ENABLED) - add_compile_definitions(PLEXDB_LOG_ENABLED=1) + if(PLEXDB_LOG_ENABLED) + add_compile_definitions(PLEXDB_LOG_ENABLED=1) + endif() + if(PLEXDB_DEBUG) + add_compile_definitions(PLEXDB_DEBUG=1) + endif() endif() + # ============================================================================= # Third part dependencies # ============================================================================= @@ -165,6 +170,10 @@ if(OBJSTORE_BUILD_SERVER) plexdb::objstore ) + target_include_directories(objstore_server PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../macros + ) + if(PLEXDB_LOG_ENABLED) target_link_options(objstore_server PRIVATE -rdynamic) endif() diff --git a/objstore/engine/dtype.cppm b/objstore/engine/dtype.cppm index 2c7a7d4..d0d1070 100644 --- a/objstore/engine/dtype.cppm +++ b/objstore/engine/dtype.cppm @@ -6,7 +6,88 @@ import plexdb.tagged_union; using namespace plexdb; +namespace { + // dynamic array + template + struct ExpandDynamicArrayHelper; + + template + struct ExpandDynamicArrayHelper> { + using type = TypeList...>; + }; + + template + using ExpandDynamicArray = ExpandDynamicArrayHelper::type; + + // dynamic map + template + struct ExpandDynamicMapValuesHelper; + + template + struct ExpandDynamicMapValuesHelper> { + using type = TypeList...>; + }; + + template + struct ExpandDynamicMapHelper; + + template + struct ExpandDynamicMapHelper, Vs> { + private: + using K1Maps = typename ExpandDynamicMapValuesHelper::type; + using K2NMaps = typename ExpandDynamicMapHelper, Vs>::type; + public: + using type = Concat; + }; + + template + struct ExpandDynamicMapHelper, Vs> { + using type = TypeList<>; + }; + + template + using ExpandDynamicMap = ExpandDynamicMapHelper::type; + + // dynamic set + template + struct ExpandDynamicSetHelper; + + template + struct ExpandDynamicSetHelper> { + using type = TypeList...>; + }; + + template + using ExpandDynamicSet = ExpandDynamicSetHelper::type; + + // tagged union + template + struct ExpandTaggedUnionHelper; + + template + struct ExpandTaggedUnionHelper> { + using type = TaggedUnion; + }; + + template + using ExpandTaggedUnion = ExpandTaggedUnionHelper::type; + + // concepts + template struct IsPairHelper { static constexpr bool is = false; }; + template struct IsPairHelper> { static constexpr bool is = true; }; + + template + concept IsPair = IsPairHelper::is; +} + export namespace objstore { + enum class CType : U8 { + native, + list, + map, + set, + }; + enum class DType : U8 { text, uuid, @@ -20,19 +101,70 @@ export namespace objstore { double_, }; + union CDType { + struct base { + CType ctype; + } base; + struct native { + CType ctype; + DType value_dtype; + } native; + struct list { + CType ctype; + DType element_dtype; + } list; + struct set { + CType ctype; + DType key_dtype; + } set; + struct map { + CType ctype; + DType key_dtype; + DType value_dtype; + } map; + }; + + constexpr bool operator==(CDType a, CDType b) { + if (a.base.ctype != b.base.ctype) return false; + switch (a.base.ctype) { + case CType::native: return a.native.value_dtype == b.native.value_dtype; + case CType::list: return a.list.element_dtype == b.list.element_dtype; + case CType::set: return a.set.key_dtype == b.set.key_dtype; + case CType::map: return a.map.key_dtype == b.map.key_dtype && a.map.value_dtype == b.map.value_dtype; + } + return false; + } + namespace dtype { - using WriteValue = TaggedUnion; - using ReadValue = TaggedUnion; - + constexpr CDType make_native(DType d) { return CDType{.native = {CType::native, d}}; } + constexpr CDType make_list(DType el) { return CDType{.list = {CType::list, el}}; } + constexpr CDType make_set(DType key) { return CDType{.set = {CType::set, key}}; } + constexpr CDType make_map(DType key, DType val) { return CDType{.map = {CType::map, key, val}}; } + + using WriteNativeTypes = TypeList; + using ReadNativeTypes = TypeList; + + using WriteNativeValue = ExpandTaggedUnion; + using WriteValue = ExpandTaggedUnion, DynamicArray>> + >>; + + using ReadValue = ExpandTaggedUnion, + ExpandDynamicMap, + ExpandDynamicSet + >>; + inline constexpr U64 hash(const WriteValue& value) { return visit(value, [](auto& v) -> U64 { using T = Decay; - if constexpr (SameAs) { - return hash(v); - } if constexpr (Either) { return os::memory_cast(&v); + } else { + return hash(v); } }); } @@ -52,9 +184,18 @@ export namespace objstore { } return "unknown"; } + inline AutoString8 to_str(CDType cdtype) { + switch (cdtype.base.ctype) { + case CType::native: return AutoString8(to_str(cdtype.native.value_dtype)); + case CType::list: return "list["_as + to_str(cdtype.list.element_dtype) + "]"; + case CType::set: return "set["_as + to_str(cdtype.set.key_dtype) + "]"; + case CType::map: return "map["_as + to_str(cdtype.map.key_dtype) + ", " + to_str(cdtype.map.value_dtype) + "]"; + } + return "unknown"_as; + } template - bool can_write(DType dtype) { + bool can_write_generic(DType dtype) { if constexpr (Either, String8>) { return dtype == DType::text || dtype == DType::timestamp || dtype == DType::uuid; } @@ -71,11 +212,46 @@ export namespace objstore { return false; } - bool can_write(const WriteValue& src, DType dtype) { + bool can_write_generic(const WriteNativeValue& src, DType dtype) { return visit(src, [&dtype](auto& v) { using T = Decay; using V = Conditional, String8, T>; - return can_write(dtype); + + return can_write_generic(dtype); + }); + } + + bool can_write_generic(const WriteValue& src, CDType cdtype) { + return visit(src, [&cdtype](auto& v) { + using T = Decay; + using V = Conditional, String8, T>; + + if constexpr (IsDynamicArray) { + if constexpr (IsPair) { + static_assert(SameAs>); + for (const auto& pair_v : v) { + if (!can_write_generic(pair_v.first, cdtype.map.key_dtype)) return false; + if (!can_write_generic(pair_v.second, cdtype.map.value_dtype)) return false; + } + return true; + } else { + static_assert(SameAs); + + if (cdtype.base.ctype == CType::list) { + for (const auto& element_v : v) { + if (!can_write_generic(element_v, cdtype.list.element_dtype)) return false; + } + return true; + } else { + for (const auto& key_v : v) { + if (!can_write_generic(key_v, cdtype.set.key_dtype)) return false; + } + return true; + } + } + } + + return can_write_generic(cdtype.native.value_dtype); }); } @@ -90,7 +266,7 @@ export namespace objstore { }; // @todo avoid copy for zero init - void write_default(const Write auto& w, DType dtype) { + void write_specific_default(const Write auto& w, DType dtype) { switch (dtype) { case DType::text: case DType::timestamp: @@ -125,77 +301,171 @@ export namespace objstore { }break; } } - void write(const Write auto& w, const String8& src, DType dtype) { - switch (dtype) { - case DType::text: - case DType::timestamp: - case DType::uuid:{ - w(reinterpret_cast(&src.length), sizeof(src.length)); - w(reinterpret_cast(src.data), src.length); - }break; - default:{ - assert_true(false, "missing implementation of String8 copy for dtype " + to_str(dtype) + ", this should never happen"); + + void write_specific_default(const Write auto& w, CDType cdtype) { + switch (cdtype.base.ctype) { + case CType::native:{ + write_specific_default(w, cdtype.native.value_dtype); + }break; + case CType::list: + case CType::set: + case CType::map:{ + U64 length = 0_u64; + w(reinterpret_cast(&length), sizeof(length)); }break; } } - void write(const Write auto& w, S64 src, DType dtype) { - switch (dtype) { - case DType::smallint:{ - S16 smallint = static_cast(src); - w(reinterpret_cast(&smallint), sizeof(smallint)); - }break; - case DType::int_:{ - S32 int_ = static_cast(src); - w(reinterpret_cast(&int_), sizeof(int_)); - }break; - case DType::counter: - case DType::bigint:{ - S64 bigint = static_cast(src); - w(reinterpret_cast(&bigint), sizeof(bigint)); - }break; - default:{ - assert_true(false, "missing implementation of S64 copy for dtype " + to_str(dtype) + ", this should never happen"); - }break; + + template + void write_generic(const Write auto& w, const T& src, DType dtype) { + if constexpr (SameAs) { + switch (dtype) { + case DType::text: + case DType::timestamp: + case DType::uuid:{ + w(reinterpret_cast(&src.length), sizeof(src.length)); + w(reinterpret_cast(src.c_str), src.length); + }break; + default:{ assert_true(false, "mismatch between underlying type and dtype"); }break; + } + } else if constexpr (SameAs) { + switch (dtype) { + case DType::smallint:{ + S16 smallint = static_cast(src); + w(reinterpret_cast(&smallint), sizeof(smallint)); + }break; + case DType::int_:{ + S32 int_ = static_cast(src); + w(reinterpret_cast(&int_), sizeof(int_)); + }break; + case DType::counter: + case DType::bigint:{ + S64 bigint = static_cast(src); + w(reinterpret_cast(&bigint), sizeof(bigint)); + }break; + default:{ assert_true(false, "mismatch between underlying type and dtype"); }break; + } + } else if constexpr (SameAs) { + switch (dtype) { + case DType::boolean:{ + U8 boolean = static_cast(src); + w(reinterpret_cast(&boolean), sizeof(boolean)); + }break; + default:{ assert_true(false, "mismatch between underlying type and dtype"); }break; + } + } else if constexpr (SameAs) { + switch (dtype) { + case DType::float_:{ + F32 float_ = static_cast(src); + w(reinterpret_cast(&float_), sizeof(float_)); + }break; + case DType::double_:{ + F64 double_ = static_cast(src); + w(reinterpret_cast(&double_), sizeof(double_)); + }break; + default:{ assert_true(false, "mismatch between underlying type and dtype"); }break; + } + } else { + static_assert(!SameAs, "missing underlying type case"); } } - void write(const Write auto& w, bool src, DType dtype) { - switch (dtype) { - case DType::boolean:{ - U8 boolean = static_cast(src); - w(reinterpret_cast(&boolean), sizeof(boolean)); - }break; - default:{ - assert_true(false, "missing implementation of bool copy for dtype " + to_str(dtype) + ", this should never happen"); - }break; + + void write_generic(const Write auto& w, const WriteNativeValue& src, DType dtype) { + visit(src, [&](const auto& v) { + write_generic(w, v, dtype); + }); + } + + void write_generic(const Write auto& w, const WriteValue& src, CDType cdtype) { + visit(src, [&cdtype, &w](const auto& v) { + using T = Decay; + + if constexpr (IsDynamicArray) { + w(reinterpret_cast(&v.length), sizeof(v.length)); + if constexpr (IsPair) { + for (const auto& el: v) { + write_generic(w, el.first, cdtype.map.key_dtype); + write_generic(w, el.second, cdtype.map.value_dtype); + } + } else { + if (cdtype.base.ctype == CType::list) { + for (const auto& el: v) write_generic(w, el, cdtype.list.element_dtype); + } else { + for (const auto& el: v) write_generic(w, el, cdtype.set.key_dtype); + } + } + } else { + write_generic(w, v, cdtype.native.value_dtype); + } + }); + } + + template + bool can_write_specific(DType dtype) { + if constexpr (Either, AutoString8>) { + return dtype == DType::text || dtype == DType::timestamp || dtype == DType::uuid; } + if constexpr (Either, S16>) { return dtype == DType::smallint; } + if constexpr (Either, S32>) { return dtype == DType::int_; } + if constexpr (Either, S64>) { return dtype == DType::bigint || dtype == DType::counter; } + if constexpr (Either, U8>) { return dtype == DType::boolean; } + if constexpr (Either, F32>) { return dtype == DType::float_; } + if constexpr (Either, F64>) { return dtype == DType::double_; } + return false; } - void write(const Write auto& w, F64 src, DType dtype) { - switch (dtype) { - case DType::float_:{ - F32 float_ = static_cast(src); - w(reinterpret_cast(&float_), sizeof(float_)); - }break; - case DType::double_:{ - F64 double_ = static_cast(src); - w(reinterpret_cast(&double_), sizeof(double_)); - }break; - default:{ - assert_true(false, "missing implementation of F64 copy for dtype " + to_str(dtype) + ", this should never happen"); - }break; + + bool can_write_specific(const ReadValue& src, CDType cdtype) { + return visit(src, [&cdtype](const auto& v) -> bool { + using T = Decay; + if constexpr (IsDynamicMap) { + if (cdtype.base.ctype != CType::map) return false; + return can_write_specific(cdtype.map.key_dtype) && + can_write_specific(cdtype.map.value_dtype); + } else if constexpr (IsDynamicSet) { + if (cdtype.base.ctype != CType::set) return false; + return can_write_specific(cdtype.set.key_dtype); + } else if constexpr (IsDynamicArray) { + if (cdtype.base.ctype != CType::list) return false; + return can_write_specific(cdtype.list.element_dtype); + } else { + return cdtype.base.ctype == CType::native && can_write_specific(cdtype.native.value_dtype); + } + }); + } + + template + void write_specific(const Write auto& w, const T& src, DType dtype) { + if constexpr (SameAs || SameAs) { + String8 s{src}; + w(reinterpret_cast(&s.length), sizeof(s.length)); + w(reinterpret_cast(s.data), s.length); + } else { + w(reinterpret_cast(&src), sizeof(src)); } } - - void write(const Write auto& w, const WriteValue& src, DType dtype) { - visit(src, [&dtype, &w](auto& v) { + + void write_specific(const Write auto& w, const ReadValue& src, CDType cdtype) { + assert_true(can_write_specific(src, cdtype), "invalid type to write for cdtype " + to_str(cdtype)); + visit(src, [&cdtype, &w](auto& v) { using T = Decay; - using V = Conditional, String8, T>; - - assert_true(can_write(dtype), "invalid type to write for dtype " + to_str(dtype)); - - if constexpr (SameAs) { - write(w, String8(v), dtype); + + if constexpr (IsDynamicMap) { + U64 len = length(v); + w(reinterpret_cast(&len), sizeof(len)); + for (auto it = v.begin(); it != v.end(); ++it) { + write_specific(w, (*it).first, cdtype.map.key_dtype); + write_specific(w, (*it).second, cdtype.map.value_dtype); + } + } else if constexpr (IsDynamicSet) { + U64 len = length(v); + w(reinterpret_cast(&len), sizeof(len)); + for (auto it = v.begin(); it != v.end(); ++it) + write_specific(w, *it, cdtype.set.key_dtype); + } else if constexpr (IsDynamicArray) { + w(reinterpret_cast(&v.length), sizeof(v.length)); + for (const auto& el: v) write_specific(w, el, cdtype.list.element_dtype); } else { - write(w, v, dtype); + write_specific(w, v, cdtype.native.value_dtype); } }); } @@ -214,7 +484,7 @@ export namespace objstore { return {move(value)}; }break; case DType::smallint:{ - S16 value; + S16 value; r(reinterpret_cast(&value), sizeof(value)); return {move(value)}; @@ -253,36 +523,6 @@ export namespace objstore { } } - void write_from_read(const Write auto& w, const ReadValue& src, DType dtype) { - switch (dtype) { - case DType::text: - case DType::uuid: - case DType::timestamp:{ - const AutoString8& s = get(src); - write(w, String8(s.c_str, s.length), dtype); - }break; - case DType::smallint: - write(w, static_cast(get(src)), dtype); - break; - case DType::int_: - write(w, static_cast(get(src)), dtype); - break; - case DType::counter: - case DType::bigint: - write(w, get(src), dtype); - break; - case DType::boolean: - write(w, static_cast(get(src)), dtype); - break; - case DType::float_: - write(w, static_cast(get(src)), dtype); - break; - case DType::double_: - write(w, get(src), dtype); - break; - } - } - AutoString8 to_str(const ReadValue& value, DType dtype) { switch (dtype) { case DType::text: @@ -311,5 +551,131 @@ export namespace objstore { }break; } } + + ReadValue read_specific(const Read auto& r, CDType cdtype) { + switch (cdtype.base.ctype) { + case CType::native: + return read(r, cdtype.native.value_dtype); + case CType::list: { + U64 length; + r(reinterpret_cast(&length), sizeof(length)); + auto read_arr = [&]() -> ReadValue { + DynamicArray arr{}; + for (U64 i = 0; i < length; i++) + push_back(arr, get(read(r, cdtype.list.element_dtype))); + return {move(arr)}; + }; + switch (cdtype.list.element_dtype) { + case DType::text: case DType::uuid: case DType::timestamp: + return read_arr.template operator()(); + case DType::smallint: return read_arr.template operator()(); + case DType::int_: return read_arr.template operator()(); + case DType::bigint: case DType::counter: + return read_arr.template operator()(); + case DType::boolean: return read_arr.template operator()(); + case DType::float_: return read_arr.template operator()(); + case DType::double_: return read_arr.template operator()(); + } + } break; + case CType::set: { + U64 length; + r(reinterpret_cast(&length), sizeof(length)); + auto read_set = [&]() -> ReadValue { + DynamicSet s{}; + for (U64 i = 0; i < length; i++) + insert(s, get(read(r, cdtype.set.key_dtype))); + return {move(s)}; + }; + switch (cdtype.set.key_dtype) { + case DType::text: case DType::uuid: case DType::timestamp: + return read_set.template operator()(); + case DType::smallint: return read_set.template operator()(); + case DType::int_: return read_set.template operator()(); + case DType::bigint: case DType::counter: + return read_set.template operator()(); + case DType::boolean: return read_set.template operator()(); + case DType::float_: return read_set.template operator()(); + case DType::double_: return read_set.template operator()(); + } + } break; + case CType::map: { + U64 length; + r(reinterpret_cast(&length), sizeof(length)); + auto read_map_kv = [&]() -> ReadValue { + DynamicMap m{}; + for (U64 i = 0; i < length; i++) { + K k = get(read(r, cdtype.map.key_dtype)); + V v = get(read(r, cdtype.map.value_dtype)); + insert(m, move(k), move(v)); + } + return {move(m)}; + }; + auto read_map_v = [&](DType val_dtype) -> ReadValue { + switch (val_dtype) { + case DType::text: case DType::uuid: case DType::timestamp: + return read_map_kv.template operator()(); + case DType::smallint: return read_map_kv.template operator()(); + case DType::int_: return read_map_kv.template operator()(); + case DType::bigint: case DType::counter: + return read_map_kv.template operator()(); + case DType::boolean: return read_map_kv.template operator()(); + case DType::float_: return read_map_kv.template operator()(); + case DType::double_: return read_map_kv.template operator()(); + } + }; + switch (cdtype.map.key_dtype) { + case DType::text: case DType::uuid: case DType::timestamp: + return read_map_v.template operator()(cdtype.map.value_dtype); + case DType::smallint: return read_map_v.template operator()(cdtype.map.value_dtype); + case DType::int_: return read_map_v.template operator()(cdtype.map.value_dtype); + case DType::bigint: case DType::counter: + return read_map_v.template operator()(cdtype.map.value_dtype); + case DType::boolean: return read_map_v.template operator()(cdtype.map.value_dtype); + case DType::float_: return read_map_v.template operator()(cdtype.map.value_dtype); + case DType::double_: return read_map_v.template operator()(cdtype.map.value_dtype); + } + } break; + } + } + + AutoString8 to_str(const ReadValue& value, CDType cdtype) { + if (cdtype.base.ctype == CType::native) + return to_str(value, cdtype.native.value_dtype); + return visit(value, [&cdtype](auto& v) -> AutoString8 { + using T = Decay; + if constexpr (IsDynamicMap) { + AutoString8 result = "{"_as; + bool first = true; + for (auto it = v.begin(); it != v.end(); ++it) { + if (!first) result = result + ", "; + result = result + to_str(ReadValue{(*it).first}, cdtype.map.key_dtype); + result = result + ": "; + result = result + to_str(ReadValue{(*it).second}, cdtype.map.value_dtype); + first = false; + } + return result + "}"; + } else if constexpr (IsDynamicSet) { + AutoString8 result = "{"_as; + bool first = true; + for (auto it = v.begin(); it != v.end(); ++it) { + if (!first) result = result + ", "; + result = result + to_str(ReadValue{*it}, cdtype.set.key_dtype); + first = false; + } + return result + "}"; + } else if constexpr (IsDynamicArray) { + AutoString8 result = "["_as; + bool first = true; + for (const auto& el : v) { + if (!first) result = result + ", "; + result = result + to_str(ReadValue{el}, cdtype.list.element_dtype); + first = false; + } + return result + "]"; + } else { + return to_str(ReadValue{v}, cdtype.native.value_dtype); + } + }); + } } } diff --git a/objstore/engine/engine.cpp b/objstore/engine/engine.cpp index 7703bc5..4c93c42 100644 --- a/objstore/engine/engine.cpp +++ b/objstore/engine/engine.cpp @@ -48,7 +48,7 @@ namespace objstore::engine { blob::get(row_blob, out_value, size, offset); offset += size; }; - return dtype::read(read, it.table->cols[it.col_idx].dtype); + return dtype::read_specific(read, it.table->cols[it.col_idx].type); } ColumnIterator& ColumnIterator::operator++() { @@ -58,7 +58,7 @@ namespace objstore::engine { blob::get(row_blob, out_value, size, offset); offset += size; }; - dtype::read(skip, table->cols[col_idx].dtype); + dtype::read_specific(skip, table->cols[col_idx].type); row_offset_bytes = offset; ++col_idx; return *this; @@ -85,6 +85,207 @@ namespace objstore::engine { return *this; } + // ======================================================================== + // system virtual tables + // ======================================================================== + bool is_system_keyspace(String8 ks) { + return ks == "system" || ks == "system_schema" || + ks == "system_virtual_schema" || ks == "system_auth" || + ks == "system_distributed" || ks == "system_traces"; + } + + VirtualRows make_system_local() { + VirtualRows vr; + vr.keyspace = "system"; + vr.table = "local"; + + push_back(vr.columns, VirtualColumn{"key", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"bootstrapped", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"broadcast_address", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"broadcast_port", dtype::make_native(DType::int_)}); + push_back(vr.columns, VirtualColumn{"cluster_name", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"cql_version", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"data_center", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"host_id", dtype::make_native(DType::uuid)}); + push_back(vr.columns, VirtualColumn{"listen_address", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"listen_port", dtype::make_native(DType::int_)}); + push_back(vr.columns, VirtualColumn{"native_protocol_version", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"partitioner", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"rack", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"release_version", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"rpc_address", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"rpc_port", dtype::make_native(DType::int_)}); + push_back(vr.columns, VirtualColumn{"schema_version", dtype::make_native(DType::uuid)}); + push_back(vr.columns, VirtualColumn{"tokens", dtype::make_set(DType::text)}); + + const U8 uuid_bytes[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; + + VirtualRow row; + push_back(row.values, dtype::ReadValue{"local"_as}); + push_back(row.values, dtype::ReadValue{"COMPLETED"_as}); + push_back(row.values, dtype::ReadValue{"127.0.0.1"_as}); + push_back(row.values, dtype::ReadValue{S32(7000)}); + push_back(row.values, dtype::ReadValue{"objstore"_as}); + push_back(row.values, dtype::ReadValue{"3.4.7"_as}); + push_back(row.values, dtype::ReadValue{"datacenter1"_as}); + push_back(row.values, dtype::ReadValue{AutoString8{uuid_bytes, 16}}); + push_back(row.values, dtype::ReadValue{"127.0.0.1"_as}); + push_back(row.values, dtype::ReadValue{S32(7000)}); + push_back(row.values, dtype::ReadValue{"4"_as}); + push_back(row.values, dtype::ReadValue{"org.apache.cassandra.dht.Murmur3Partitioner"_as}); + push_back(row.values, dtype::ReadValue{"rack1"_as}); + push_back(row.values, dtype::ReadValue{"3.11.19"_as}); // @note last version in 3.x, before system_virtual + push_back(row.values, dtype::ReadValue{"127.0.0.1"_as}); + push_back(row.values, dtype::ReadValue{S32(9042)}); + push_back(row.values, dtype::ReadValue{AutoString8{uuid_bytes, 16}}); + push_back(row.values, dtype::ReadValue{DynamicSet{{"0"_as}}}); + push_back(vr.rows, move(row)); + + return vr; + } + + VirtualRows make_system_peers() { + VirtualRows vr; + vr.keyspace = "system"; + vr.table = "peers"; + + push_back(vr.columns, VirtualColumn{"peer", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"data_center", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"host_id", dtype::make_native(DType::uuid)}); + push_back(vr.columns, VirtualColumn{"preferred_ip", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"rack", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"release_version", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"rpc_address", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"schema_version", dtype::make_native(DType::uuid)}); + push_back(vr.columns, VirtualColumn{"tokens", dtype::make_set(DType::text)}); + + return vr; + } + + VirtualRows make_system_peers_v2() { + VirtualRows vr; + vr.keyspace = "system"; + vr.table = "peers_v2"; + + push_back(vr.columns, VirtualColumn{"peer", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"peer_port", dtype::make_native(DType::int_)}); + push_back(vr.columns, VirtualColumn{"data_center", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"host_id", dtype::make_native(DType::uuid)}); + push_back(vr.columns, VirtualColumn{"native_address", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"native_port", dtype::make_native(DType::int_)}); + push_back(vr.columns, VirtualColumn{"preferred_ip", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"preferred_port", dtype::make_native(DType::int_)}); + push_back(vr.columns, VirtualColumn{"rack", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"release_version", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"schema_version", dtype::make_native(DType::uuid)}); + push_back(vr.columns, VirtualColumn{"tokens", dtype::make_native(DType::text)}); + + return vr; + } + + VirtualRows make_schema_keyspaces(schema::Schema& schema) { + VirtualRows vr; + vr.keyspace = "system_schema"; + vr.table = "keyspaces"; + + push_back(vr.columns, VirtualColumn{"keyspace_name", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"durable_writes", dtype::make_native(DType::boolean)}); + push_back(vr.columns, VirtualColumn{"replication", dtype::make_native(DType::text)}); + + for (auto& ks : schema.keyspaces) { + if (ks.tombstone) continue; + VirtualRow row; + push_back(row.values, dtype::ReadValue{AutoString8(ks.name)}); + push_back(row.values, dtype::ReadValue{U8(1)}); + push_back(row.values, dtype::ReadValue{"{'class': 'SimpleStrategy', 'replication_factor': '1'}"_as}); + push_back(vr.rows, move(row)); + } + + return vr; + } + + VirtualRows make_schema_tables(schema::Schema& schema) { + VirtualRows vr; + vr.keyspace = "system_schema"; + vr.table = "tables"; + + push_back(vr.columns, VirtualColumn{"keyspace_name", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"table_name", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"bloom_filter_fp_chance", dtype::make_native(DType::double_)}); + push_back(vr.columns, VirtualColumn{"comment", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"default_time_to_live", dtype::make_native(DType::int_)}); + push_back(vr.columns, VirtualColumn{"gc_grace_seconds", dtype::make_native(DType::int_)}); + + for (auto& ks : schema.keyspaces) { + if (ks.tombstone) continue; + for (auto& tbl : ks.tbls) { + if (tbl.tombstone) continue; + VirtualRow row; + push_back(row.values, dtype::ReadValue{AutoString8(ks.name)}); + push_back(row.values, dtype::ReadValue{AutoString8(tbl.name)}); + push_back(row.values, dtype::ReadValue{F64(0.01)}); + push_back(row.values, dtype::ReadValue{""_as}); + push_back(row.values, dtype::ReadValue{S32(0)}); + push_back(row.values, dtype::ReadValue{S32(864000)}); + push_back(vr.rows, move(row)); + } + } + + return vr; + } + + VirtualRows make_schema_columns(schema::Schema& schema) { + VirtualRows vr; + vr.keyspace = "system_schema"; + vr.table = "columns"; + + push_back(vr.columns, VirtualColumn{"keyspace_name", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"table_name", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"column_name", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"clustering_order", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"kind", dtype::make_native(DType::text)}); + push_back(vr.columns, VirtualColumn{"position", dtype::make_native(DType::int_)}); + push_back(vr.columns, VirtualColumn{"type", dtype::make_native(DType::text)}); + + for (auto& ks : schema.keyspaces) { + if (ks.tombstone) continue; + for (auto& tbl : ks.tbls) { + if (tbl.tombstone) continue; + S32 pos = 0; + for (U64 ci = 0; ci < tbl.cols.length; ci++) { + auto& col = tbl.cols[ci]; + if (col.tombstone) continue; + VirtualRow row; + push_back(row.values, dtype::ReadValue{AutoString8(ks.name)}); + push_back(row.values, dtype::ReadValue{AutoString8(tbl.name)}); + push_back(row.values, dtype::ReadValue{AutoString8(col.name)}); + push_back(row.values, dtype::ReadValue{"none"_as}); + bool is_partition_key = (ci == tbl.primary_col_idx); + push_back(row.values, dtype::ReadValue{is_partition_key ? "partition_key"_as : "regular"_as}); + push_back(row.values, dtype::ReadValue{S32(is_partition_key ? 0 : pos++)}); + push_back(row.values, dtype::ReadValue{AutoString8(dtype::to_str(col.type))}); + push_back(vr.rows, move(row)); + } + } + } + + return vr; + } + + Optional try_system_select(Engine& engine, String8 keyspace, String8 table) { + if (keyspace == "system") { + if (table == "local") return make_system_local(); + if (table == "peers") return make_system_peers(); + if (table == "peers_v2") return make_system_peers_v2(); + } + if (keyspace == "system_schema") { + if (table == "keyspaces") return make_schema_keyspaces(engine.schema); + if (table == "tables") return make_schema_tables(engine.schema); + if (table == "columns") return make_schema_columns(engine.schema); + } + return {}; + } + // ======================================================================== // execute // ======================================================================== @@ -93,6 +294,16 @@ namespace objstore::engine { using T = RemoveCVRef; if constexpr (SameAs) { + if (is_system_keyspace(stmt.keyspace_name)) { + if (stmt.if_not_exists) { + return {.status = ExecutionStatus::Success, .kind = ResultKind::Void}; + } + return { + .status = ExecutionStatus::AlreadyExists, + .message = "Keyspace already exists", + .keyspace = stmt.keyspace_name, + }; + } if (!stmt.if_not_exists) { auto existing = schema::read_keyspace(engine.schema, stmt.keyspace_name); if (existing != nullptr) { @@ -188,7 +399,7 @@ namespace objstore::engine { const auto& col = tbl->cols[idx]; const auto& value = stmt.values[idx]; - if (!dtype::can_write(value, col.dtype)) { + if (!dtype::can_write_generic(value, col.type)) { return { .status = ExecutionStatus::Invalid, .message = "Value does not match its column's type", @@ -212,7 +423,7 @@ namespace objstore::engine { const auto& col = tbl->cols[idx]; const auto& value = stmt.values[idx]; - dtype::write(write, value, col.dtype); + dtype::write_generic(write, value, col.type); } // @todo uniqueness check @@ -220,6 +431,17 @@ namespace objstore::engine { return {.status = ExecutionStatus::Success, .kind = ResultKind::Void}; } else if constexpr (SameAs) { + auto system_vr = try_system_select(engine, stmt.keyspace_name, stmt.table_name); + if (system_vr) { + return { + .status = ExecutionStatus::Success, + .kind = ResultKind::VirtualRows, + .keyspace = stmt.keyspace_name, + .table = stmt.table_name, + .virtual_rows = move(system_vr), + }; + } + // @todo implement these features assert_true(stmt.column_names.cap == 0, "SELECT with column names not implemented"); assert_true(stmt.where.cap == 0, "SELECT with WHERE clause not implemented"); @@ -269,6 +491,10 @@ namespace objstore::engine { .rows = move(rows), }; } else if constexpr (SameAs) { + if (is_system_keyspace(stmt.keyspace_name)) { + engine.current_keyspace = stmt.keyspace_name; + return {.status = ExecutionStatus::Success, .kind = ResultKind::UseKeyspace, .keyspace = engine.current_keyspace}; + } auto ks = schema::read_keyspace(engine.schema, stmt.keyspace_name); if (ks == nullptr) { return { @@ -343,7 +569,7 @@ namespace objstore::engine { if (stmt.op == AlterTableOp::add_column) { for (U64 i = 0; i < stmt.columns.length; i++) { const auto& col = stmt.columns[i]; - CreateColumn create{.name = col.name, .dtype = col.dtype, .is_primary_key = false}; + CreateColumn create{.name = col.name, .type = col.type, .is_primary_key = false}; if (schema::create_column(engine.schema, *tbl, create) == nullptr) { return { .status = ExecutionStatus::Invalid, @@ -481,7 +707,7 @@ namespace objstore::engine { read_offset += size; }; for (const auto& col : tbl->cols) { - push_back(read_values, dtype::read(read_fn, col.dtype)); + push_back(read_values, dtype::read_specific(read_fn, col.type)); } // rewrite row applying assignments @@ -502,9 +728,9 @@ namespace objstore::engine { } } if (updated != nullptr) { - dtype::write(write_fn, *updated, col.dtype); + dtype::write_generic(write_fn, *updated, col.type); } else { - dtype::write_from_read(write_fn, read_values[ci], col.dtype); + dtype::write_specific(write_fn, read_values[ci], col.type); } } @@ -568,7 +794,7 @@ namespace objstore::engine { read_offset += size; }; for (const auto& col : tbl->cols) { - push_back(read_values, dtype::read(read_fn, col.dtype)); + push_back(read_values, dtype::read_specific(read_fn, col.type)); } blob::resize(row_blob, 0); @@ -587,9 +813,9 @@ namespace objstore::engine { } } if (is_deleted) { - dtype::write_default(write_fn, col.dtype); + dtype::write_specific_default(write_fn, col.type); } else { - dtype::write_from_read(write_fn, read_values[ci], col.dtype); + dtype::write_specific(write_fn, read_values[ci], col.type); } } } diff --git a/objstore/engine/engine.cppm b/objstore/engine/engine.cppm index 5feda49..51bbc12 100644 --- a/objstore/engine/engine.cppm +++ b/objstore/engine/engine.cppm @@ -53,10 +53,30 @@ export namespace objstore::engine { enum class ResultKind : U8 { Void = 0, // No result (INSERT, UPDATE, DELETE) Rows, // SELECT result + VirtualRows, // Virtual/system table result (not backed by storage) SchemaChange, // CREATE/DROP/ALTER result UseKeyspace, // USE keyspace }; + // ======================================================================== + // virtual rows (for system/virtual tables not backed by storage) + // ======================================================================== + struct VirtualColumn { + String8 name; + CDType type; + }; + + struct VirtualRow { + DynamicArray values; + }; + + struct VirtualRows { + String8 keyspace; + String8 table; + DynamicArray columns; + DynamicArray rows; + }; + // ======================================================================== // row/column iterators // ======================================================================== @@ -113,6 +133,7 @@ export namespace objstore::engine { String8 table = ""; Optional rows; + Optional virtual_rows; }; ExecutionResult execute(Engine& engine, const Statement& statement); diff --git a/objstore/engine/schema.cpp b/objstore/engine/schema.cpp index 4146492..3f5beda 100644 --- a/objstore/engine/schema.cpp +++ b/objstore/engine/schema.cpp @@ -111,7 +111,7 @@ namespace objstore::schema { Column col { .tombstone = col_storage.header.tombstone, .name = col_storage.name, - .dtype = col_storage.header.dtype, + .type = col_storage.header.type, }; push_back(tbl.cols, move(col)); @@ -331,7 +331,7 @@ namespace objstore::schema { .header = ColumnHeader{ .tombstone = false, .name_length = create.name.length, - .dtype = create.dtype, + .type = create.type, .table_idx = tbl.idx, }, .name = AutoString8(create.name), @@ -352,7 +352,7 @@ namespace objstore::schema { Column col { .tombstone = col_storage_ref.header.tombstone, .name = col_storage_ref.name, - .dtype = col_storage_ref.header.dtype, + .type = col_storage_ref.header.type, }; return &push_back(tbl.cols, move(col)); } diff --git a/objstore/engine/schema.cppm b/objstore/engine/schema.cppm index 77da808..f87ae92 100644 --- a/objstore/engine/schema.cppm +++ b/objstore/engine/schema.cppm @@ -15,7 +15,7 @@ export namespace objstore::schema { struct Column { bool tombstone; String8 name; - const DType dtype; + const CDType type; }; struct Table { @@ -53,7 +53,7 @@ export namespace objstore::schema { struct ColumnHeader { bool tombstone; U64 name_length; - DType dtype; + CDType type; U64 table_idx; }; #pragma pack(pop) diff --git a/objstore/engine/statements.cppm b/objstore/engine/statements.cppm index 1d8254a..64af25e 100644 --- a/objstore/engine/statements.cppm +++ b/objstore/engine/statements.cppm @@ -111,7 +111,7 @@ export namespace objstore { // ======================================================================== struct CreateColumn { String8 name; - DType dtype; + CDType type; bool is_primary_key = false; }; @@ -133,7 +133,7 @@ export namespace objstore { struct AlterColumn { String8 name; - DType dtype; // only used for add_column + CDType type; // only used for add_column }; struct RenameColumn { diff --git a/objstore/entry/main.cpp b/objstore/entry/main.cpp index cb2354e..596d9b0 100644 --- a/objstore/entry/main.cpp +++ b/objstore/entry/main.cpp @@ -1,3 +1,5 @@ +#include "macros.h" + import plexdb.base; import plexdb.os; import plexdb.pager; @@ -12,8 +14,13 @@ using namespace objstore; using namespace plexdb; void assert_handler(const char* msg, const char* file_name, const char* function_name, unsigned line_number) { - println(msg); - os::signal_exit(1); + println("assert failed with message \"", msg, "\""); + + #if PLEXDB_DEBUG + PLEXDB_TRAP; + #else + os::signal_exit(1); + #endif } os::Notifier g_signal_notifier{}; diff --git a/objstore/http/http.cppm b/objstore/http/http.cppm index 72de84a..432cef2 100644 --- a/objstore/http/http.cppm +++ b/objstore/http/http.cppm @@ -110,9 +110,70 @@ namespace objstore::http { append(str, '"'); } + template concept IsHttpDA = IsDynamicArray; + template concept IsHttpDS = IsDynamicSet; + template concept IsHttpDM = IsDynamicMap; + + // Serialize a single native collection element as a JSON value + template + void append_json_native_element(BufferedString8& str, const T& v) { + using TT = Decay; + if constexpr (SameAs) { + append_json_text(str, String8(v.c_str, v.length)); + } else if constexpr (SameAs) { + append(str, v); + } else if constexpr (SameAs) { + append(str, v); + } else if constexpr (SameAs) { + append(str, v); + } else if constexpr (SameAs) { + append(str, static_cast(v)); + } else if constexpr (SameAs) { + append(str, v); + } else if constexpr (SameAs) { + append(str, v); + } + } + template - void append_json_dtype_value(BufferedString8& str, const dtype::ReadValue& value, DType dtype) { - switch (dtype) { + void append_json_dtype_value(BufferedString8& str, const dtype::ReadValue& value, CDType cdtype) { + if (cdtype.base.ctype != CType::native) { + visit(value, [&](const auto& v) { + using T = Decay; + if constexpr (IsHttpDM) { + // map → JSON object (numeric keys are quoted as strings) + append(str, '{'); + bool first = true; + for (auto it = v.begin(); it != v.end(); ++it) { + if (!first) append(str, ','); + first = false; + using Key = Decay; + if constexpr (SameAs) { + append_json_text(str, String8((*it).first.c_str, (*it).first.length)); + } else { + append(str, '"'); + append(str, (*it).first); + append(str, '"'); + } + append(str, ':'); + append_json_native_element(str, (*it).second); + } + append(str, '}'); + } else if constexpr (IsHttpDS || IsHttpDA) { + // list/set → JSON array + append(str, '['); + bool first = true; + for (auto it = v.begin(); it != v.end(); ++it) { + if (!first) append(str, ','); + first = false; + append_json_native_element(str, *it); + } + append(str, ']'); + } + }); + return; + } + switch (cdtype.native.value_dtype) { case DType::text: case DType::uuid: case DType::timestamp:{ @@ -172,6 +233,9 @@ namespace objstore::http { case engine::ResultKind::Rows:{ assert_true(false, "unexpected execution result"); }break; + case engine::ResultKind::VirtualRows:{ + assert_true(false, "unexpected execution result"); + }break; } } } @@ -228,7 +292,7 @@ export namespace objstore::http { if (col_idx != 0) append(response_bstr, ','); - append_json_dtype_value(response_bstr, read_value(col), column(col).dtype); + append_json_dtype_value(response_bstr, read_value(col), column(col).type); ++col_idx; } append(response_bstr, ']'); diff --git a/objstore/log/log.cpp b/objstore/log/log.cpp index d671b82..799a521 100644 --- a/objstore/log/log.cpp +++ b/objstore/log/log.cpp @@ -13,4 +13,12 @@ namespace objstore::log { query.length ); } + + void cql_parse_error(const char* text, plexdb::U64 len) { + plexdb::log::fire_message( + producer.id, + text, + len + ); + } } diff --git a/objstore/log/log.cppm b/objstore/log/log.cppm index 4ce5619..1d6fd68 100644 --- a/objstore/log/log.cppm +++ b/objstore/log/log.cppm @@ -5,4 +5,5 @@ import plexdb.log; export namespace objstore::log { void cql_parse(plexdb::String8 query); + void cql_parse_error(const char* text, plexdb::U64 len); } diff --git a/objstore/native/native.cppm b/objstore/native/native.cppm index d9190c5..1d18107 100644 --- a/objstore/native/native.cppm +++ b/objstore/native/native.cppm @@ -60,6 +60,9 @@ namespace objstore::native { constexpr U16 Uuid = 0x000C; constexpr U16 Varchar = 0x000D; constexpr U16 Smallint = 0x0013; + constexpr U16 List = 0x0020; + constexpr U16 Map = 0x0021; + constexpr U16 Set = 0x0022; } constexpr U16 dtype_to_cql_type(DType dtype) { @@ -78,6 +81,15 @@ namespace objstore::native { return 0x0000; } + constexpr U16 dtype_to_cql_type(CDType cdtype) { + if (cdtype.base.ctype != CType::native) return 0x0000; + return dtype_to_cql_type(cdtype.native.value_dtype); + } + + template concept IsCqlDA = IsDynamicArray; + template concept IsCqlDS = IsDynamicSet; + template concept IsCqlDM = IsDynamicMap; + // ======================================================================== // Per-connection state // ======================================================================== @@ -134,10 +146,115 @@ namespace objstore::native { } } - // Write a dtype::ReadValue as CQL native binary [bytes] (big-endian network order) + // Write a single native element as [bytes] (big-endian, includes the [int] length prefix) + template + void append_cql_native_element(BufferedString8& buf, const T& v) { + using TT = Decay; + if constexpr (SameAs) { + append_cql_bytes_raw(buf, reinterpret_cast(v.c_str), S32(v.length)); + } else if constexpr (SameAs) { + S64 vv = v; + U8 data[8]; + for (int i = 7; i >= 0; i--) { data[i] = U8(vv); vv >>= 8; } + append_cql_bytes_raw(buf, data, 8); + } else if constexpr (SameAs) { + U8 data[4] = { U8(U32(v) >> 24), U8(U32(v) >> 16), U8(U32(v) >> 8), U8(U32(v)) }; + append_cql_bytes_raw(buf, data, 4); + } else if constexpr (SameAs) { + U8 data[2] = { U8(U16(v) >> 8), U8(v) }; + append_cql_bytes_raw(buf, data, 2); + } else if constexpr (SameAs) { + append_cql_bytes_raw(buf, &v, 1); + } else if constexpr (SameAs) { + U32 bits; os::memory_copy(&bits, &v, sizeof(bits)); + U8 data[4] = { U8(bits >> 24), U8(bits >> 16), U8(bits >> 8), U8(bits) }; + append_cql_bytes_raw(buf, data, 4); + } else if constexpr (SameAs) { + U64 bits; os::memory_copy(&bits, &v, sizeof(bits)); + U8 data[8]; + for (int i = 7; i >= 0; i--) { data[i] = U8(bits); bits >>= 8; } + append_cql_bytes_raw(buf, data, 8); + } + } + + template + S32 cql_native_element_byte_size(const T& v) { + using TT = Decay; + if constexpr (SameAs) return S32(v.length); + if constexpr (SameAs) return 8; + if constexpr (SameAs) return 4; + if constexpr (SameAs) return 2; + if constexpr (SameAs) return 1; + if constexpr (SameAs) return 4; + if constexpr (SameAs) return 8; + return 0; + } + + // Write an [option] for a column type (§4.2.5.2): [short] type_id + optional nested type(s) + template + void append_cql_type_option(BufferedString8& buf, CDType cdtype) { + switch (cdtype.base.ctype) { + case CType::native: + append_be_u16(buf, dtype_to_cql_type(cdtype.native.value_dtype)); + break; + case CType::list: + append_be_u16(buf, cql_type::List); + append_be_u16(buf, dtype_to_cql_type(cdtype.list.element_dtype)); + break; + case CType::set: + append_be_u16(buf, cql_type::Set); + append_be_u16(buf, dtype_to_cql_type(cdtype.set.key_dtype)); + break; + case CType::map: + append_be_u16(buf, cql_type::Map); + append_be_u16(buf, dtype_to_cql_type(cdtype.map.key_dtype)); + append_be_u16(buf, dtype_to_cql_type(cdtype.map.value_dtype)); + break; + } + } + + // Write a dtype::ReadValue as CQL binary [bytes] (big-endian network order) template - void append_cql_value(BufferedString8& buf, const dtype::ReadValue& value, DType dtype) { - switch (dtype) { + void append_cql_value(BufferedString8& buf, const dtype::ReadValue& value, CDType cdtype) { + if (cdtype.base.ctype != CType::native) { + // Serialize collection as [bytes]: [int] body_len | [int] elem_count | n×[bytes] + visit(value, [&](const auto& v) { + using T = Decay; + if constexpr (IsCqlDM) { + U64 pair_count = length(v); + S32 body = 4; // [int] pair count + for (auto it = v.begin(); it != v.end(); ++it) { + body += 4 + cql_native_element_byte_size((*it).first); + body += 4 + cql_native_element_byte_size((*it).second); + } + append_be_s32(buf, body); + append_be_s32(buf, S32(pair_count)); + for (auto it = v.begin(); it != v.end(); ++it) { + append_cql_native_element(buf, (*it).first); + append_cql_native_element(buf, (*it).second); + } + } else if constexpr (IsCqlDS) { + U64 elem_count = length(v); + S32 body = 4; + for (auto it = v.begin(); it != v.end(); ++it) + body += 4 + cql_native_element_byte_size(*it); + append_be_s32(buf, body); + append_be_s32(buf, S32(elem_count)); + for (auto it = v.begin(); it != v.end(); ++it) + append_cql_native_element(buf, *it); + } else if constexpr (IsCqlDA) { + S32 body = 4; + for (U64 i = 0; i < v.length; i++) + body += 4 + cql_native_element_byte_size(v[i]); + append_be_s32(buf, body); + append_be_s32(buf, S32(v.length)); + for (U64 i = 0; i < v.length; i++) + append_cql_native_element(buf, v[i]); + } + }); + return; + } + switch (cdtype.native.value_dtype) { case DType::text: case DType::uuid: case DType::timestamp: { @@ -266,14 +383,36 @@ namespace objstore::native { for (U64 ci = 0; ci < tbl->cols.length; ci++) { append_cql_string(buf, tbl->cols[ci].name); - append_be_u16(buf, dtype_to_cql_type(tbl->cols[ci].dtype)); + append_cql_type_option(buf, tbl->cols[ci].type); } append_be_s32(buf, S32(btree::size(tbl->btree))); for (auto& row = result.rows->begin(); row != result.rows->end(); ++row) { for (auto col = engine::columns_begin(row); col != engine::columns_end(row); ++col) - append_cql_value(buf, engine::read_value(col), engine::column(col).dtype); + append_cql_value(buf, engine::read_value(col), engine::column(col).type); + } + } + + template + void append_result_virtual_rows(BufferedString8& buf, engine::VirtualRows& vr) { + append_be_s32(buf, result_kind::ROWS); + + append_be_s32(buf, 0x0001); // Global_tables_spec flag + append_be_s32(buf, S32(vr.columns.length)); + append_cql_string(buf, vr.keyspace); + append_cql_string(buf, vr.table); + + for (U64 ci = 0; ci < vr.columns.length; ci++) { + append_cql_string(buf, vr.columns[ci].name); + append_cql_type_option(buf, vr.columns[ci].type); + } + + append_be_s32(buf, S32(vr.rows.length)); + + for (U64 ri = 0; ri < vr.rows.length; ri++) { + for (U64 ci = 0; ci < vr.columns.length; ci++) + append_cql_value(buf, vr.rows[ri].values[ci], vr.columns[ci].type); } } @@ -355,6 +494,11 @@ namespace objstore::native { auto frame = make_native_frame(conn, &chunk, write, opcode::RESULT, stream); append_result_rows(frame, result, tbl); }break; + case engine::ResultKind::VirtualRows:{ + assert_true(result.virtual_rows.has_value(), "virtual rows missing"); + auto frame = make_native_frame(conn, &chunk, write, opcode::RESULT, stream); + append_result_virtual_rows(frame, *result.virtual_rows); + }break; } } break; diff --git a/objstore/native/native.test.cpp b/objstore/native/native.test.cpp index 4d68d0d..9b1fef6 100644 --- a/objstore/native/native.test.cpp +++ b/objstore/native/native.test.cpp @@ -438,3 +438,204 @@ TEST_CASE("Native protocol data persists across restarts", "[objstore.native]") srv.join(); } } + +TEST_CASE("Native protocol system.local virtual view", "[objstore.native]") { + int port = get_unique_port(); + os::File db_file{os::file_tmp()}; + REQUIRE(!os::is_zero_handle(db_file)); + + os::Notifier signal_pipe; + std::binary_semaphore server_ready{0}; + volatile bool exit_signal = false; + + std::thread server_thread([port, &signal_pipe, &server_ready, &exit_signal, &db_file]() { + U64 page_size = 4_kb; + pager::create(db_file, page_size); + Pager pager{db_file}; + engine::create_database(pager); + engine::Engine engine{&pager}; + native::run(port, signal_pipe, exit_signal, engine, [&server_ready]() { + server_ready.release(); + }); + }); + + server_ready.acquire(); + Socket client{socket_open()}; + socket_set_timeout(client, 2000); + REQUIRE(socket_connect(client, "127.0.0.1", (U16)port)); + + send_frame(client, make_startup()); + REQUIRE(recv_frame(client).opcode == 0x02); + + SECTION("SELECT * FROM system.local returns rows with expected columns") { + send_frame(client, make_query("SELECT * FROM system.local;")); + auto resp = recv_frame(client); + REQUIRE(resp.opcode == 0x08); + CHECK(result_kind(resp) == 0x0002); + CHECK(body_contains(resp, "local")); + CHECK(body_contains(resp, "datacenter1")); + CHECK(body_contains(resp, "rack1")); + CHECK(body_contains(resp, "objstore")); + } + + SECTION("SELECT * FROM system.peers returns empty rows") { + send_frame(client, make_query("SELECT * FROM system.peers;")); + auto resp = recv_frame(client); + REQUIRE(resp.opcode == 0x08); + CHECK(result_kind(resp) == 0x0002); + CHECK(body_contains(resp, "peer")); + } + + SECTION("SELECT * FROM system.peers_v2 returns empty rows") { + send_frame(client, make_query("SELECT * FROM system.peers_v2;")); + auto resp = recv_frame(client); + REQUIRE(resp.opcode == 0x08); + CHECK(result_kind(resp) == 0x0002); + } + + exit_signal = true; + os::signal_notify_safe(signal_pipe); + server_thread.join(); +} + +TEST_CASE("Native protocol system_schema virtual views", "[objstore.native]") { + int port = get_unique_port(); + os::File db_file{os::file_tmp()}; + REQUIRE(!os::is_zero_handle(db_file)); + + os::Notifier signal_pipe; + std::binary_semaphore server_ready{0}; + volatile bool exit_signal = false; + + std::thread server_thread([port, &signal_pipe, &server_ready, &exit_signal, &db_file]() { + U64 page_size = 4_kb; + pager::create(db_file, page_size); + Pager pager{db_file}; + engine::create_database(pager); + engine::Engine engine{&pager}; + native::run(port, signal_pipe, exit_signal, engine, [&server_ready]() { + server_ready.release(); + }); + }); + + server_ready.acquire(); + Socket client{socket_open()}; + socket_set_timeout(client, 2000); + REQUIRE(socket_connect(client, "127.0.0.1", (U16)port)); + + send_frame(client, make_startup()); + REQUIRE(recv_frame(client).opcode == 0x02); + + send_frame(client, make_query("CREATE KEYSPACE test_ks WITH replication = 'SimpleStrategy';")); + REQUIRE(recv_frame(client).opcode == 0x08); + + send_frame(client, make_query("CREATE TABLE test_ks.users (id int PRIMARY KEY, name text);")); + REQUIRE(recv_frame(client).opcode == 0x08); + + // system_schema.keyspaces + send_frame(client, make_query("SELECT * FROM system_schema.keyspaces;")); + { + auto resp = recv_frame(client); + REQUIRE(resp.opcode == 0x08); + CHECK(result_kind(resp) == 0x0002); + CHECK(body_contains(resp, "keyspace_name")); + CHECK(body_contains(resp, "test_ks")); + } + + // system_schema.tables + send_frame(client, make_query("SELECT * FROM system_schema.tables;")); + { + auto resp = recv_frame(client); + REQUIRE(resp.opcode == 0x08); + CHECK(result_kind(resp) == 0x0002); + CHECK(body_contains(resp, "table_name")); + CHECK(body_contains(resp, "users")); + } + + // system_schema.columns + send_frame(client, make_query("SELECT * FROM system_schema.columns;")); + { + auto resp = recv_frame(client); + REQUIRE(resp.opcode == 0x08); + CHECK(result_kind(resp) == 0x0002); + CHECK(body_contains(resp, "column_name")); + CHECK(body_contains(resp, "partition_key")); + CHECK(body_contains(resp, "name")); + } + + send_frame(client, make_query("DROP TABLE test_ks.users;")); + CHECK(recv_frame(client).opcode == 0x08); + send_frame(client, make_query("DROP KEYSPACE test_ks;")); + CHECK(recv_frame(client).opcode == 0x08); + + exit_signal = true; + os::signal_notify_safe(signal_pipe); + server_thread.join(); +} + +namespace { + bool body_contains_bytes(const NativeResponse& resp, std::initializer_list needle) { + const auto* data = resp.body.data(); + size_t size = resp.body.size(); + size_t n = needle.size(); + if (n == 0 || size < n) return false; + const uint8_t* pat = needle.begin(); + for (size_t i = 0; i + n <= size; i++) { + if (memcmp(data + i, pat, n) == 0) return true; + } + return false; + } +} + +TEST_CASE("Native protocol collection serialization", "[objstore.native]") { + // system.local has a tokens column of type set with one element "0". + // This test verifies the CQL native binary encoding of collection types and values. + int port = get_unique_port(); + os::File db_file{os::file_tmp()}; + REQUIRE(!os::is_zero_handle(db_file)); + + os::Notifier signal_pipe; + std::binary_semaphore server_ready{0}; + volatile bool exit_signal = false; + + std::thread server_thread([port, &signal_pipe, &server_ready, &exit_signal, &db_file]() { + U64 page_size = 4_kb; + pager::create(db_file, page_size); + Pager pager{db_file}; + engine::create_database(pager); + engine::Engine engine{&pager}; + native::run(port, signal_pipe, exit_signal, engine, [&server_ready]() { + server_ready.release(); + }); + }); + + server_ready.acquire(); + Socket client{socket_open()}; + socket_set_timeout(client, 2000); + REQUIRE(socket_connect(client, "127.0.0.1", (U16)port)); + + send_frame(client, make_startup()); + REQUIRE(recv_frame(client).opcode == 0x02); + + send_frame(client, make_query("SELECT * FROM system.local;")); + auto resp = recv_frame(client); + REQUIRE(resp.opcode == 0x08); + REQUIRE(result_kind(resp) == 0x0002); // Rows + + SECTION("tokens column type option is Set") { + // [short] 0x0022 (Set) + [short] 0x000D (Varchar) + CHECK(body_contains_bytes(resp, {0x00, 0x22, 0x00, 0x0D})); + } + + SECTION("tokens set value encodes one-element set containing '0'") { + // [int] body=9 | [int] count=1 | [int] elemlen=1 | '0'(0x30) + CHECK(body_contains_bytes(resp, {0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, + 0x30})); + } + + exit_signal = true; + os::signal_notify_safe(signal_pipe); + server_thread.join(); +} diff --git a/objstore/parsers/parsers.cpp b/objstore/parsers/parsers.cpp index d73733f..25007d0 100644 --- a/objstore/parsers/parsers.cpp +++ b/objstore/parsers/parsers.cpp @@ -4,10 +4,9 @@ module; #include #include #include +#include #include - -// @todo -#include +#include #define LEXY_LIT_CI(x) lexy::dsl::ascii::case_folding(LEXY_LIT(x)) @@ -473,47 +472,47 @@ namespace objstore::parsers::cql { struct data_type { struct text : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("text"); - static constexpr auto value = lexy::constant(DType::text); + static constexpr auto value = lexy::constant(dtype::make_native(DType::text)); }; struct int_ : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("int"); - static constexpr auto value = lexy::constant(DType::int_); + static constexpr auto value = lexy::constant(dtype::make_native(DType::int_)); }; struct bigint : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("bigint"); - static constexpr auto value = lexy::constant(DType::bigint); + static constexpr auto value = lexy::constant(dtype::make_native(DType::bigint)); }; struct smallint : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("smallint"); - static constexpr auto value = lexy::constant(DType::smallint); + static constexpr auto value = lexy::constant(dtype::make_native(DType::smallint)); }; struct counter : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("counter"); - static constexpr auto value = lexy::constant(DType::counter); + static constexpr auto value = lexy::constant(dtype::make_native(DType::counter)); }; struct timestamp : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("timestamp"); - static constexpr auto value = lexy::constant(DType::timestamp); + static constexpr auto value = lexy::constant(dtype::make_native(DType::timestamp)); }; struct boolean : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("boolean"); - static constexpr auto value = lexy::constant(DType::boolean); + static constexpr auto value = lexy::constant(dtype::make_native(DType::boolean)); }; struct float_ : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("float"); - static constexpr auto value = lexy::constant(DType::float_); + static constexpr auto value = lexy::constant(dtype::make_native(DType::float_)); }; struct double_ : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("double"); - static constexpr auto value = lexy::constant(DType::double_); + static constexpr auto value = lexy::constant(dtype::make_native(DType::double_)); }; struct uuid : lexy::transparent_production { static constexpr auto rule = LEXY_LIT_CI("uuid"); - static constexpr auto value = lexy::constant(DType::uuid); + static constexpr auto value = lexy::constant(dtype::make_native(DType::uuid)); }; static constexpr auto rule = dsl::p | dsl::p | dsl::p | dsl::p | dsl::p | dsl::p | dsl::p | dsl::p | dsl::p | dsl::p; - static constexpr auto value = lexy::forward; + static constexpr auto value = lexy::forward; }; // string literal @@ -952,7 +951,7 @@ namespace objstore::parsers::cql { struct column_def { static constexpr auto rule = [] { return (dsl::member<&CreateColumn::name> = dsl::p) + dsl::p + - (dsl::member<&CreateColumn::dtype> = dsl::p) + dsl::p + + (dsl::member<&CreateColumn::type> = dsl::p) + dsl::p + (dsl::member<&CreateColumn::is_primary_key> = dsl::p) + dsl::p; }(); @@ -1153,7 +1152,7 @@ namespace objstore::parsers::cql { auto select = dsl::p; return dsl::p + (use | create_ks | alter_ks | drop_ks | create_tbl | drop_tbl | - truncate | insert | update | del | select) + dsl::p + dsl::lit_c<';'>; + truncate | insert | update | del | select) + dsl::p + (dsl::lit_c<';'> | dsl::eof); }(); static constexpr auto value = lexy::construct; @@ -1165,20 +1164,66 @@ namespace objstore::parsers::cql { }; } - // @todo return error message + struct LogErrorCallback { + struct _sink { + using return_type = size_t; + size_t _count = 0; + + template + void operator()(const lexy::error_context& context, + const lexy::error& error) { + auto location = lexy::get_input_location(context.input(), error.position()); + + AutoString8 msg; + if constexpr (SameAs) { + msg = fmt("error: while parsing %s at %u:%u: expected '%.*s'", + context.production(), + location.line_nr(), location.column_nr(), + static_cast(error.length()), reinterpret_cast(error.string())); + } else if constexpr (SameAs) { + msg = fmt("error: while parsing %s at %u:%u: expected keyword '%.*s'", + context.production(), + location.line_nr(), location.column_nr(), + static_cast(error.length()), reinterpret_cast(error.string())); + } else if constexpr (SameAs) { + msg = fmt("error: while parsing %s at %u:%u: expected %s", + context.production(), + location.line_nr(), location.column_nr(), + error.name()); + } else { + msg = fmt("error: while parsing %s at %u:%u: %s", + context.production(), + location.line_nr(), location.column_nr(), + error.message()); + } + + log::cql_parse_error(msg.c_str, msg.length); + ++_count; + } + + size_t finish() && { + return _count; + } + }; + + auto sink() const { + return _sink{}; + } + }; + Optional parse(String8 bytes, bool report_errors) { log::cql_parse(bytes); auto input = lexy::string_input(bytes.data, bytes.length); - // auto reporter = lexy_ext::report_error.path("cql"); - auto reporter = lexy::noop; - auto result = lexy::parse(input, reporter); + auto try_parse = [&](auto callback) -> Optional { + auto result = lexy::parse(input, callback); + if (result.has_value()) return result.value(); + return {}; + }; - if (result.has_value()) { - return result.value(); - } - return {}; + if (report_errors) return try_parse(LogErrorCallback{}); + return try_parse(lexy::noop); } bool is_complete(String8 bytes) { diff --git a/objstore/parsers/parsers.cppm b/objstore/parsers/parsers.cppm index 428a3f3..397a9fe 100644 --- a/objstore/parsers/parsers.cppm +++ b/objstore/parsers/parsers.cppm @@ -1,6 +1,7 @@ export module objstore.parsers; import plexdb.base; +import plexdb.log; import plexdb.tagged_union; import objstore.engine.dtype; import objstore.engine.statements; @@ -122,7 +123,7 @@ namespace objstore::parsers { // cassandra query language (CQL) // ======================================================================== namespace cql { - export Optional parse(String8 bytes, bool stderr=false); + export Optional parse(String8 bytes, bool report_errors=log::enabled); export bool is_complete(String8 bytes); } } diff --git a/objstore/parsers/parsers.test.cpp b/objstore/parsers/parsers.test.cpp index cefe0fb..34eeaba 100644 --- a/objstore/parsers/parsers.test.cpp +++ b/objstore/parsers/parsers.test.cpp @@ -289,7 +289,7 @@ TEST_CASE("CQL CREATE TABLE statements", "[objstore.parser]") { REQUIRE(tbl.if_not_exists == false); REQUIRE(tbl.columns.length == 1); REQUIRE(tbl.columns[0].name == "id"); - REQUIRE(tbl.columns[0].dtype == DType::int_); + REQUIRE(tbl.columns[0].type == dtype::make_native(DType::int_)); REQUIRE(tbl.columns[0].is_primary_key == true); } @@ -305,15 +305,15 @@ TEST_CASE("CQL CREATE TABLE statements", "[objstore.parser]") { REQUIRE(tbl.columns.length == 3); REQUIRE(tbl.columns[0].name == "id"); - REQUIRE(tbl.columns[0].dtype == DType::int_); + REQUIRE(tbl.columns[0].type == dtype::make_native(DType::int_)); REQUIRE(tbl.columns[0].is_primary_key == true); REQUIRE(tbl.columns[1].name == "name"); - REQUIRE(tbl.columns[1].dtype == DType::text); + REQUIRE(tbl.columns[1].type == dtype::make_native(DType::text)); REQUIRE(tbl.columns[1].is_primary_key == false); REQUIRE(tbl.columns[2].name == "age"); - REQUIRE(tbl.columns[2].dtype == DType::int_); + REQUIRE(tbl.columns[2].type == dtype::make_native(DType::int_)); REQUIRE(tbl.columns[2].is_primary_key == false); } @@ -337,11 +337,11 @@ TEST_CASE("CQL CREATE TABLE statements", "[objstore.parser]") { REQUIRE(result.has_value()); const auto& tbl = get(result->value); REQUIRE(tbl.columns.length == 5); - REQUIRE(tbl.columns[0].dtype == DType::int_); - REQUIRE(tbl.columns[1].dtype == DType::text); - REQUIRE(tbl.columns[2].dtype == DType::bigint); - REQUIRE(tbl.columns[3].dtype == DType::timestamp); - REQUIRE(tbl.columns[4].dtype == DType::boolean); + REQUIRE(tbl.columns[0].type == dtype::make_native(DType::int_)); + REQUIRE(tbl.columns[1].type == dtype::make_native(DType::text)); + REQUIRE(tbl.columns[2].type == dtype::make_native(DType::bigint)); + REQUIRE(tbl.columns[3].type == dtype::make_native(DType::timestamp)); + REQUIRE(tbl.columns[4].type == dtype::make_native(DType::boolean)); } SECTION("CREATE TABLE with FLOAT and DOUBLE types") { @@ -351,8 +351,8 @@ TEST_CASE("CQL CREATE TABLE statements", "[objstore.parser]") { REQUIRE(result.has_value()); const auto& tbl = get(result->value); REQUIRE(tbl.columns.length == 3); - REQUIRE(tbl.columns[1].dtype == DType::float_); - REQUIRE(tbl.columns[2].dtype == DType::double_); + REQUIRE(tbl.columns[1].type == dtype::make_native(DType::float_)); + REQUIRE(tbl.columns[2].type == dtype::make_native(DType::double_)); } SECTION("CREATE TABLE with UUID type") { @@ -362,7 +362,7 @@ TEST_CASE("CQL CREATE TABLE statements", "[objstore.parser]") { REQUIRE(result.has_value()); const auto& tbl = get(result->value); REQUIRE(tbl.columns.length == 2); - REQUIRE(tbl.columns[0].dtype == DType::uuid); + REQUIRE(tbl.columns[0].type == dtype::make_native(DType::uuid)); REQUIRE(tbl.columns[0].is_primary_key == true); } @@ -639,18 +639,6 @@ TEST_CASE("CQL Invalid syntax handling", "[objstore.parser]") { auto result = cql::parse(query); REQUIRE_FALSE(result.has_value()); } - - SECTION("Missing semicolon from SELECT FROM") { - auto query = "SELECT * FROM my_app.products"; - auto result = cql::parse(query); - REQUIRE_FALSE(result.has_value()); - } - - SECTION("Missing semicolon from CREATE KEYSPACE") { - auto query = "CREATE KEYSPACE no_semi WITH replication = 'test'"; - auto result = cql::parse(query); - REQUIRE_FALSE(result.has_value()); - } SECTION("Missing table name in CREATE TABLE") { auto query = "CREATE TABLE;"; @@ -1115,4 +1103,31 @@ TEST_CASE("Parse CREATE KEYSPACE with map literal replication", "[objstore.parse REQUIRE(type_matches_tag(ks.options[1].value)); REQUIRE(get(ks.options[1].value) == "true"); } +} + +TEST_CASE("CQL parse error reporting", "[objstore.parser]") { + SECTION("Invalid syntax with report_errors returns empty") { + auto result = cql::parse("INVALID STATEMENT;", true); + REQUIRE_FALSE(result.has_value()); + } + + SECTION("Valid query with report_errors succeeds") { + auto result = cql::parse("SELECT * FROM ks.tbl;", true); + REQUIRE(result.has_value()); + } + + SECTION("Empty query with report_errors returns empty") { + auto result = cql::parse("", true); + REQUIRE_FALSE(result.has_value()); + } + + SECTION("Unclosed string with report_errors returns empty") { + auto result = cql::parse("INSERT INTO ks.tbl VALUES ('unclosed);", true); + REQUIRE_FALSE(result.has_value()); + } + + SECTION("Multiple error types with report_errors returns empty") { + auto result = cql::parse("CREATE TABLE;", true); + REQUIRE_FALSE(result.has_value()); + } } \ No newline at end of file diff --git a/objstore/repl/repl.cpp b/objstore/repl/repl.cpp index 053c4c4..7c9a055 100644 --- a/objstore/repl/repl.cpp +++ b/objstore/repl/repl.cpp @@ -53,6 +53,12 @@ namespace objstore::repl { } os::stream_write(ostream, "\n"); }break; + case engine::ResultKind::VirtualRows:{ + AutoString8 count_str = to_str(row_count); + os::stream_write(ostream, "("); + os::stream_write(ostream, count_str.c_str, count_str.length); + os::stream_write(ostream, " rows)\n"); + }break; } } @@ -111,7 +117,31 @@ namespace objstore::repl { U64 col_idx = 0; for (auto col = engine::columns_begin(row); col != engine::columns_end(row); ++col, ++col_idx) { os::stream_write(ostream, col_idx == 0 ? " " : " | "); - AutoString8 val_str = dtype::to_str(engine::read_value(col), engine::column(col).dtype); + AutoString8 val_str = dtype::to_str(engine::read_value(col), engine::column(col).type); + os::stream_write(ostream, val_str.c_str, val_str.length); + } + os::stream_write(ostream, "\n"); + ++row_count; + } + } else if (result.virtual_rows) { + auto& vr = *result.virtual_rows; + if (vr.columns.length > 0) { + for (U64 ci = 0; ci < vr.columns.length; ci++) { + if (ci > 0) os::stream_write(ostream, " | "); + os::stream_write(ostream, vr.columns[ci].name); + } + os::stream_write(ostream, "\n"); + for (U64 ci = 0; ci < vr.columns.length; ci++) { + if (ci > 0) os::stream_write(ostream, "-+-"); + for (U64 j = 0; j < vr.columns[ci].name.length; j++) + os::stream_write(ostream, "-", 1); + } + os::stream_write(ostream, "\n"); + } + for (U64 ri = 0; ri < vr.rows.length; ri++) { + for (U64 ci = 0; ci < vr.columns.length; ci++) { + os::stream_write(ostream, ci == 0 ? " " : " | "); + AutoString8 val_str = dtype::to_str(vr.rows[ri].values[ci], vr.columns[ci].type); os::stream_write(ostream, val_str.c_str, val_str.length); } os::stream_write(ostream, "\n"); diff --git a/objstore/test/assert_helpers.test.cpp b/objstore/test/assert_helpers.test.cpp index 4f04b03..2c7a108 100644 --- a/objstore/test/assert_helpers.test.cpp +++ b/objstore/test/assert_helpers.test.cpp @@ -3,6 +3,7 @@ import plexdb.base; inline void catch2_assert_handler(const char* msg, const char* file_name, const char* function_name, unsigned line_number) { + // @todo do not throw, just register exception and continue until test case ends, use macro+destructor to fail FAIL("Assert failed \"" << msg << "\" at " << function_name << " in " << file_name << ":" << line_number); } diff --git a/plexdb/CMakeLists.txt b/plexdb/CMakeLists.txt index ec0f4e3..2ac9ba4 100644 --- a/plexdb/CMakeLists.txt +++ b/plexdb/CMakeLists.txt @@ -14,6 +14,14 @@ project(plexdb) if(PLEXDB_IS_MAIN_PROJECT) option(BUILD_TESTS "Build unit tests" ON) option(PLEXDB_LOG_ENABLED "Enable structured logging hooks" OFF) + option(PLEXDB_DEBUG "Build debug checks" ON) + + if(PLEXDB_DEBUG) + add_compile_definitions(PLEXDB_DEBUG=1) + endif() + if(PLEXDB_LOG_ENABLED) + add_compile_definitions(PLEXDB_LOG_ENABLED=1) + endif() endif() # ============================================================================= @@ -58,6 +66,7 @@ target_sources(plexdb_modules btree/btree.types.cppm os/containers.cppm os/core.cppm + third_party/xxhash/xxhash.cppm os/function.cppm os/os.cppm os/signal.cppm @@ -110,10 +119,6 @@ target_sources(plexdb target_link_libraries(plexdb PUBLIC plexdb_modules) -if(PLEXDB_LOG_ENABLED) - target_compile_definitions(plexdb_modules PUBLIC PLEXDB_LOG_ENABLED=1) -endif() - # ============================================================================= # Tests # ============================================================================= diff --git a/plexdb/base/string.cpp b/plexdb/base/string.cpp index b9fd0c4..dcc5af5 100644 --- a/plexdb/base/string.cpp +++ b/plexdb/base/string.cpp @@ -11,6 +11,7 @@ module; module plexdb.base.string; import plexdb.os; +import xxhash; namespace plexdb { String8::String8(const AutoString8& str) : data(str.c_str), length(str.length) {} @@ -118,6 +119,22 @@ namespace plexdb { bool AutoString8::operator==(const char* b) const { return strcmp(this->c_str, b) == 0; } + bool AutoString8::operator==(const AutoString8& b) const { + if (this->length != b.length) return false; + for (U64 i = 0; i < this->length; i++) { + if (this->c_str[i] != b.c_str[i]) return false; + } + return true; + } + U64 hash(const AutoString8& s) { + U64 result = XXHash64::hash(s.c_str, s.length, 0); + return result | (result == 0); + } + + U64 hash(String8 s) { + U64 result = XXHash64::hash(s.data, s.length, 0); + return result | (result == 0); + } void AutoString8::push_back(const char& c) { plexdb::push_back(*this, c); diff --git a/plexdb/base/string.cppm b/plexdb/base/string.cppm index 14f52e4..2ccad8b 100644 --- a/plexdb/base/string.cppm +++ b/plexdb/base/string.cppm @@ -74,6 +74,7 @@ export namespace plexdb { bool operator==(const String8& b) const; bool operator==(const char* b) const; + bool operator==(const AutoString8& b) const; // stl helpers void push_back(const char& c); @@ -85,6 +86,9 @@ export namespace plexdb { void append(AutoString8& str, const char* first, const char* last); void append(AutoString8& str, String8 s); + U64 hash(const AutoString8& s); + U64 hash(String8 s); + AutoString8 operator+(const String8& lhs, const String8& rhs); AutoString8 operator""_as(const char* str, size_t len); diff --git a/plexdb/base/types.cppm b/plexdb/base/types.cppm index 3814b14..eccd004 100644 --- a/plexdb/base/types.cppm +++ b/plexdb/base/types.cppm @@ -196,15 +196,20 @@ export namespace plexdb { PLEXDB_CONSTEVAL_TRAP(expr); } else { if (unlikely(!expr)) { - PLEXDB_TRAP; - if (g_assert_handler != nullptr) + if (g_assert_handler != nullptr) { g_assert_handler(msg, loc.file_name(), loc.function_name(), loc.line()); + } else { + PLEXDB_TRAP; + } } } } constexpr inline void assert_true(bool expr, const char* msg, std::source_location loc = std::source_location::current()) noexcept { - if constexpr (!k_assert_enabled && !PLEXDB_IS_CONSTEVAL()) - return; + if constexpr (!k_assert_enabled) { + if (!PLEXDB_IS_CONSTEVAL()) { + return; + } + } assert_true_always(expr, msg, loc); } @@ -341,6 +346,9 @@ export namespace plexdb { // ======================================================================== template struct Pair { + using First = A; + using Second = B; + A first{}; B second{}; @@ -802,4 +810,36 @@ namespace plexdb { export template using Decay = typename DecayHelper::type; + + // ======================================================================== + // type combinatorics + // ======================================================================== + export template + struct TypeList {}; + + template + struct ConcatHelper; + + template<> + struct ConcatHelper<> { + using type = TypeList<>; + }; + + template + struct ConcatHelper { + using type = List; + }; + + template + struct ConcatHelper { + using type = typename ConcatHelper::type, Rest...>::type; + }; + + template + struct ConcatHelper, TypeList> { + using type = TypeList; + }; + + export template + using Concat = typename ConcatHelper::type; } \ No newline at end of file diff --git a/plexdb/os/containers.cppm b/plexdb/os/containers.cppm index 6cdac44..1ae9723 100644 --- a/plexdb/os/containers.cppm +++ b/plexdb/os/containers.cppm @@ -1,10 +1,18 @@ // @todo benchmark +module; +#include export module plexdb.os.containers; import plexdb.base; import plexdb.os.core; export namespace plexdb { + // ======================================================================== + // dynamic array + // ======================================================================== + constexpr U64 DYNAMIC_ARRAY_INITIAL_CAPACITY = 4; + constexpr U64 DYNAMIC_ARRAY_CAPACITY_GROWTH_RATE = 2; + template struct DynamicArray; @@ -20,6 +28,8 @@ export namespace plexdb { // @todo move construct vs zero initialize template struct DynamicArray { + using Element = T; + T* ptr = nullptr; Size length = 0; Size capacity = 0; @@ -108,6 +118,12 @@ export namespace plexdb { // stl helpers void push_back(const T& value) { plexdb::push_back(*this, value); } + + DynamicArray(std::initializer_list init) { + reserve(*this, init.size()); + for (const T& v : init) + plexdb::push_back(*this, v); + } }; template @@ -130,10 +146,22 @@ export namespace plexdb { arr.capacity = new_capacity; } + template + void resize(DynamicArray& arr, Size new_length) { + reserve(arr, new_length); + + // default construct new memory + for (Size i = 0; i < new_length; ++i) { + new (arr.ptr + i) T(); + } + + arr.length = new_length; + } + template void reserve_for_push(DynamicArray& arr) { if (arr.length == arr.capacity) { - Size new_capacity = arr.capacity == 0 ? 4 : arr.capacity * 2; + Size new_capacity = arr.capacity == 0 ? DYNAMIC_ARRAY_INITIAL_CAPACITY : arr.capacity * DYNAMIC_ARRAY_CAPACITY_GROWTH_RATE; reserve(arr, new_capacity); } } @@ -188,4 +216,477 @@ export namespace plexdb { arr.length = 0; } + + // ======================================================================== + // map + // ======================================================================== + constexpr U64 DYNAMIC_MAP_MAX_PAIRS_PER_SLOT = 4; + constexpr U64 DYNAMIC_MAP_INITIAL_SLOTS = 4; + constexpr U64 DYNAMIC_MAP_SLOT_GROWTH_RATE = 2; + + template + struct DynamicMap; + + template + bool find_slot_and_pair(const DynamicMap& map, const K& key, U64& slot_idx, U64& pair_idx); + + template + void reserve_for_push(DynamicMap& map); + + template + V& insert(DynamicMap& map, const K& key, const V& value); + + template + struct DynamicMap { + using Key = K; + using Value = V; + + DynamicArray>> slots; + + struct Iterator { + DynamicMap* map = nullptr; + U64 slot_idx = 0; + U64 pair_idx = 0; + + bool operator!=(const Iterator& other) const { + return map != other.map || slot_idx != other.slot_idx || pair_idx != other.pair_idx; + } + + Pair& operator*() { + return map->slots[slot_idx][pair_idx]; + } + + Iterator& operator++() { + if (!map) return *this; + pair_idx++; + while (slot_idx < map->slots.length && pair_idx >= map->slots[slot_idx].length) { + slot_idx++; + pair_idx = 0; + } + if (slot_idx >= map->slots.length) { map = nullptr; slot_idx = 0; } + return *this; + } + + }; + struct ConstIterator { + const DynamicMap* map = nullptr; + U64 slot_idx = 0; + U64 pair_idx = 0; + + bool operator!=(const ConstIterator& other) const { + return map != other.map || slot_idx != other.slot_idx || pair_idx != other.pair_idx; + } + + const Pair& operator*() const { + return map->slots[slot_idx][pair_idx]; + } + + ConstIterator& operator++() { + if (!map) return *this; + pair_idx++; + while (slot_idx < map->slots.length && pair_idx >= map->slots[slot_idx].length) { + slot_idx++; + pair_idx = 0; + } + if (slot_idx >= map->slots.length) { map = nullptr; slot_idx = 0; } + return *this; + } + }; + + Iterator begin() { + U64 s = 0; + while (s < slots.length && slots[s].length == 0) s++; + if (s >= slots.length) return Iterator{ .map = nullptr }; + return Iterator{ .map = this, .slot_idx = s, .pair_idx = 0 }; + } + + Iterator end() { + return Iterator{ .map = nullptr }; + } + + ConstIterator begin() const { + U64 s = 0; + while (s < slots.length && slots[s].length == 0) s++; + if (s >= slots.length) return ConstIterator{ .map = nullptr }; + return ConstIterator{ .map = this, .slot_idx = s, .pair_idx = 0 }; + } + + ConstIterator end() const { + return ConstIterator{ .map = nullptr }; + } + + Iterator operator[](const K& key) noexcept { + U64 slot_idx, pair_idx; + if (find_slot_and_pair(*this, key, slot_idx, pair_idx)) { + return Iterator{ .map = this, .slot_idx = slot_idx, .pair_idx = pair_idx }; + } + + reserve_for_push(*this); + slot_idx = hash(key) % slots.length; + DynamicArray>& bucket = slots[slot_idx]; + push_back(bucket, Pair(key, V{})); + pair_idx = bucket.length - 1; + return Iterator{ .map = this, .slot_idx = slot_idx, .pair_idx = pair_idx }; + } + + ConstIterator operator[](const K& key) const noexcept { + U64 slot_idx, pair_idx; + find_slot_and_pair(*this, key, slot_idx, pair_idx); + return ConstIterator{ .map = const_cast(this), .slot_idx = slot_idx, .pair_idx = pair_idx }; + } + + DynamicMap(std::initializer_list> init) { + for (const Pair& p : init) + insert(*this, p.first, p.second); + } + }; + + template + void rehash(DynamicMap& map, U64 new_slot_count) { + DynamicArray>> old_slots = map.slots; + + DynamicArray>> new_slots; + resize(new_slots, new_slot_count); + for (U64 i = 0; i < new_slot_count; i++) { + reserve(new_slots[i], DYNAMIC_MAP_MAX_PAIRS_PER_SLOT); + } + + for (U64 i = 0; i < old_slots.length; i++) { + DynamicArray>& bucket = old_slots[i]; + for (U64 j = 0; j < bucket.length; j++) { + const Pair& p = bucket[j]; + U64 new_idx = hash(p.first) % new_slot_count; + new_slots[new_idx].push_back(Pair(p.first, p.second)); + } + } + + map.slots = move(new_slots); + } + + template + void reserve_for_push(DynamicMap& map) { + if (map.slots.length == 0) { + resize(map.slots, DYNAMIC_MAP_INITIAL_SLOTS); + return; + } + + for (U64 i = 0; i < map.slots.length; i++) { + if (map.slots[i].length > DYNAMIC_MAP_MAX_PAIRS_PER_SLOT) { + U64 new_slot_count = map.slots.length * DYNAMIC_MAP_SLOT_GROWTH_RATE + 1; + rehash(map, new_slot_count); + break; + } + } + } + + template + bool find_slot_and_pair(const DynamicMap& map, const K& key, U64& slot_idx, U64& pair_idx) { + slot_idx = hash(key) % map.slots.length; + const DynamicArray>& bucket = map.slots[slot_idx]; + for (U64 i = 0; i < bucket.length; i++) { + if (bucket[i].first == key) { + pair_idx = i; + return true; + } + } + return false; + } + + template + U64 length(const DynamicMap& map) { + U64 n = 0; + for (U64 i = 0; i < map.slots.length; i++) n += map.slots[i].length; + return n; + } + + template + typename DynamicMap::Iterator find_it(DynamicMap& map, const K& key) { + U64 slot_idx, pair_idx; + if (!find_slot_and_pair(map, key, slot_idx, pair_idx)) return map.end(); + return typename DynamicMap::Iterator{ .map = &map, .slot_idx = slot_idx, .pair_idx = pair_idx }; + } + + template + typename DynamicMap::ConstIterator find_it(const DynamicMap& map, const K& key) { + U64 slot_idx, pair_idx; + if (!find_slot_and_pair(map, key, slot_idx, pair_idx)) return map.end(); + return typename DynamicMap::ConstIterator{ .map = &map, .slot_idx = slot_idx, .pair_idx = pair_idx }; + } + + template + V* find(DynamicMap& map, const K& key) { + U64 slot_idx, pair_idx; + if (!find_slot_and_pair(map, key, slot_idx, pair_idx)) return nullptr; + return &map.slots[slot_idx][pair_idx]; + } + + template + const V* find(const DynamicMap& map, const K& key) { + U64 slot_idx, pair_idx; + if (!find_slot_and_pair(map, key, slot_idx, pair_idx)) return nullptr; + return &map.slots[slot_idx][pair_idx]; + } + + template + V& insert(DynamicMap& map, const K& key, const V& value) { + U64 slot_idx, pair_idx; + if (find_slot_and_pair(map, key, slot_idx, pair_idx)) { + map.slots[slot_idx][pair_idx].second = value; + return map.slots[slot_idx][pair_idx].second; + } + + reserve_for_push(map); + slot_idx = hash(key) % map.slots.length; + DynamicArray>& bucket = map.slots[slot_idx]; + push_back(bucket, Pair(key, value)); + return bucket[bucket.length - 1].second; + } + + template + V& emplace(DynamicMap& map, const K& key, Args&&... args) { + U64 slot_idx, pair_idx; + if (find_slot_and_pair(map, key, slot_idx, pair_idx)) { + return map.slots[slot_idx][pair_idx].second; + } + + reserve_for_push(map); + slot_idx = hash(key) % map.slots.length; + DynamicArray>& bucket = map.slots[slot_idx]; + push_back(bucket, Pair(key, V(Forward(args)...))); + return bucket[bucket.length - 1].second; + } + + template + typename DynamicMap::Iterator clear(DynamicMap& map) { + for (U64 i = 0; i < map.slots.length; i++) { + clear(map.slots[i]); + } + return map.end(); + } + + template + void reserve(DynamicMap& map, U64 count) { + reserve(map.slots, 1 + count/DYNAMIC_MAP_MAX_PAIRS_PER_SLOT); + for (U64 i = 0; i < map.slots.length; i++) { + reserve(map.slots[i], DYNAMIC_MAP_MAX_PAIRS_PER_SLOT); + } + } + + // ======================================================================== + // set + // ======================================================================== + constexpr U64 DYNAMIC_SET_MAX_KEYS_PER_SLOT = 4; + constexpr U64 DYNAMIC_SET_INITIAL_SLOTS = 4; + constexpr U64 DYNAMIC_SET_SLOT_GROWTH_RATE = 2; + + template + struct DynamicSet; + + template + bool find_slot(const DynamicSet& set, const K& key, U64& slot_idx, U64& key_idx); + + template + void reserve_for_push(DynamicSet& set); + + template + bool insert(DynamicSet& set, const K& key); + + template + struct DynamicSet { + using Key = K; + DynamicArray> slots; + + struct Iterator { + DynamicSet* set = nullptr; + U64 slot_idx = 0; + U64 key_idx = 0; + + bool operator!=(const Iterator& other) const { + return set != other.set || slot_idx != other.slot_idx || key_idx != other.key_idx; + } + + K& operator*() { + return set->slots[slot_idx][key_idx]; + } + + Iterator& operator++() { + if (!set) return *this; + key_idx++; + while (slot_idx < set->slots.length && key_idx >= set->slots[slot_idx].length) { + slot_idx++; + key_idx = 0; + } + if (slot_idx >= set->slots.length) { set = nullptr; slot_idx = 0; } + return *this; + } + }; + + struct ConstIterator { + const DynamicSet* set = nullptr; + U64 slot_idx = 0; + U64 key_idx = 0; + + bool operator!=(const ConstIterator& other) const { + return set != other.set || slot_idx != other.slot_idx || key_idx != other.key_idx; + } + + const K& operator*() const { + return set->slots[slot_idx][key_idx]; + } + + ConstIterator& operator++() { + if (!set) return *this; + key_idx++; + while (slot_idx < set->slots.length && key_idx >= set->slots[slot_idx].length) { + slot_idx++; + key_idx = 0; + } + if (slot_idx >= set->slots.length) { set = nullptr; slot_idx = 0; } + return *this; + } + }; + + Iterator begin() { + U64 s = 0; + while (s < slots.length && slots[s].length == 0) s++; + if (s >= slots.length) return Iterator{ nullptr }; + return Iterator{ this, s, 0 }; + } + + Iterator end() { + return Iterator{ nullptr }; + } + + ConstIterator begin() const { + U64 s = 0; + while (s < slots.length && slots[s].length == 0) s++; + if (s >= slots.length) return ConstIterator{ nullptr }; + return ConstIterator{ const_cast(this), s, 0 }; + } + + ConstIterator end() const { + return ConstIterator{ .set = nullptr }; + } + + Iterator operator[](const K& key) { + U64 slot_idx, key_idx; + if (find_slot(*this, key, slot_idx, key_idx)) { + return Iterator{ this, slot_idx, key_idx }; + } + + reserve_for_push(*this); + slot_idx = hash(key) % slots.length; + DynamicArray& bucket = slots[slot_idx]; + push_back(bucket, key); + key_idx = bucket.length - 1; + return Iterator{ this, slot_idx, key_idx }; + } + + ConstIterator operator[](const K& key) const { + U64 slot_idx, key_idx; + find_slot(*this, key, slot_idx, key_idx); + return ConstIterator{ const_cast(this), slot_idx, key_idx }; + } + + DynamicSet(std::initializer_list init) { + for (const K& k : init) + insert(*this, k); + } + }; + + template + bool find_slot(const DynamicSet& set, const K& key, U64& slot_idx, U64& key_idx) { + if (set.slots.length == 0) return false; + slot_idx = hash(key) % set.slots.length; + const DynamicArray& bucket = set.slots[slot_idx]; + for (U64 i = 0; i < bucket.length; i++) { + if (bucket[i] == key) { + key_idx = i; + return true; + } + } + return false; + } + + template + void reserve_for_push(DynamicSet& set) { + if (set.slots.length == 0) { + resize(set.slots, DYNAMIC_SET_INITIAL_SLOTS); + return; + } + + for (U64 i = 0; i < set.slots.length; i++) { + if (set.slots[i].length > DYNAMIC_SET_MAX_KEYS_PER_SLOT) { + U64 new_slot_count = set.slots.length * DYNAMIC_SET_SLOT_GROWTH_RATE + 1; + rehash(set, new_slot_count); + break; + } + } + } + + template + void rehash(DynamicSet& set, U64 new_slot_count) { + DynamicArray> old_slots = move(set.slots); + DynamicArray> new_slots; + resize(new_slots, new_slot_count); + + for (U64 i = 0; i < new_slot_count; i++) { + reserve(new_slots[i], DYNAMIC_SET_MAX_KEYS_PER_SLOT); + } + + for (U64 i = 0; i < old_slots.length; i++) { + DynamicArray& bucket = old_slots[i]; + for (U64 j = 0; j < bucket.length; j++) { + const K& key = bucket[j]; + U64 idx = hash(key) % new_slot_count; + push_back(new_slots[idx], key); + } + } + + set.slots = move(new_slots); + } + + template + U64 length(const DynamicSet& set) { + U64 n = 0; + for (U64 i = 0; i < set.slots.length; i++) n += set.slots[i].length; + return n; + } + + template + bool contains(const DynamicSet& set, const K& key) { + U64 slot_idx, key_idx; + return find_slot(set, key, slot_idx, key_idx); + } + + template + bool insert(DynamicSet& set, const K& key) { + U64 slot_idx, key_idx; + if (find_slot(set, key, slot_idx, key_idx)) return false; + reserve_for_push(set); + slot_idx = hash(key) % set.slots.length; + push_back(set.slots[slot_idx], key); + return true; + } + + template + void clear(DynamicSet& set) { + for (U64 i = 0; i < set.slots.length; i++) { + clear(set.slots[i]); + } + } + + // ======================================================================== + // Container type concepts + // ======================================================================== + template struct IsDynamicArrayHelper { static constexpr bool value = false; }; + template struct IsDynamicArrayHelper> { static constexpr bool value = true; }; + template struct IsDynamicSetHelper { static constexpr bool value = false; }; + template struct IsDynamicSetHelper> { static constexpr bool value = true; }; + template struct IsDynamicMapHelper { static constexpr bool value = false; }; + template struct IsDynamicMapHelper> { static constexpr bool value = true; }; + + template concept IsDynamicArray = IsDynamicArrayHelper::value; + template concept IsDynamicSet = IsDynamicSetHelper::value; + template concept IsDynamicMap = IsDynamicMapHelper::value; } \ No newline at end of file diff --git a/plexdb/third_party/xxhash/xxhash.cppm b/plexdb/third_party/xxhash/xxhash.cppm index 6aacfde..9722036 100644 --- a/plexdb/third_party/xxhash/xxhash.cppm +++ b/plexdb/third_party/xxhash/xxhash.cppm @@ -1,4 +1,4 @@ -module: +module; #include export module xxhash; @@ -23,7 +23,7 @@ export module xxhash; Note: my code is NOT endian-aware ! **/ -class XXHash64 +export class XXHash64 { public: /// create new XXHash (64 bit)