From 52734e074dcd998baeea81611eb59ceb6d55d4e2 Mon Sep 17 00:00:00 2001 From: kould Date: Tue, 3 Nov 2026 23:30:29 +0800 Subject: [PATCH 01/14] fix(mark-apply): rebuild the materialized RHS when an executor reruns A reused executor, such as a Program's, resets its tree before every execution, and MarkApply kept its materialized RHS through that reset, so an IN/ANY subquery answered with the rows of an earlier execution, possibly from another transaction. Executor::start now marks the reset as a restart, and MarkApply keeps its cache only when a parent rescans it within one execution. --- src/db/program.rs | 33 +++++++++++++++++++++++++++++++++ src/execution/dql/mark_apply.rs | 2 +- src/execution/mod.rs | 14 +++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/db/program.rs b/src/db/program.rs index 16b4ac10..99c14fd1 100644 --- a/src/db/program.rs +++ b/src/db/program.rs @@ -817,5 +817,38 @@ mod tests { } Ok(()) } + + #[test] + fn rerun_rebuilds_materialized_subquery() -> Result<(), DatabaseError> { + let mut db = DataBaseBuilder::path(".").build_in_memory()?; + db.ddl("create table t1 (c1 int primary key, name varchar)")?; + db.ddl("create table t2 (c1 int primary key, name varchar)")?; + db.run("insert into t1 values (0, 'a'), (1, 'b'), (2, 'c'), (3, 'd')")? + .done()?; + let mut plan = + db.program_sql("select c1 from t1 where name in (select name from t2)", &[])?; + + // Each execution runs in its own transaction and must see what the previous one missed. + for (insert, expected) in [ + ("insert into t2 values (0, 'b')", vec![1]), + ("insert into t2 values (1, 'd')", vec![1, 3]), + ] { + db.run(insert)?.done()?; + let mut tx = db.new_transaction(true)?; + let mut iter = plan.execute(&mut tx, [])?; + let mut rows = Vec::new(); + while let Some(c1) = iter.next_tuple(|_, row| row.value_at(0).clone())? { + rows.push(c1); + } + iter.done()?; + tx.commit()?; + let expected = expected + .into_iter() + .map(DataValue::Int32) + .collect::>(); + assert_eq!(rows, expected); + } + Ok(()) + } } // GRCOV_EXCL_STOP diff --git a/src/execution/dql/mark_apply.rs b/src/execution/dql/mark_apply.rs index a68427b4..dc48fecb 100644 --- a/src/execution/dql/mark_apply.rs +++ b/src/execution/dql/mark_apply.rs @@ -126,7 +126,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { ) -> Result<(), DatabaseError> { // Materialization already makes the RHS reusable across left rows. // Keep both complete caches and paused prefixes; only reset the left input. - if !self.op.materialize || self.right_input.is_none() { + if arena.restarting() || !self.op.materialize || self.right_input.is_none() { self.right_input = Some(RightInput::new(self.op.materialize)); } self.left_input.reset(arena, plan_arena) diff --git a/src/execution/mod.rs b/src/execution/mod.rs index ffaf29b9..7204af28 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -186,8 +186,11 @@ impl<'a> Executor<'a> { ) -> Result<(), DatabaseError> { self.buffer.reset(); if mem::replace(&mut self.ran, true) { + self.buffer.restarting = true; let mut context = ExecContext::new(&mut self.buffer, arena); - self.root.reset(&mut context, plan_arena)?; + let result = self.root.reset(&mut context, plan_arena); + self.buffer.restarting = false; + result?; } Ok(()) } @@ -247,6 +250,9 @@ pub(crate) enum ExecNode<'a> { pub(crate) trait ExecutorNode<'a, T: StorageTransaction + 'a>: Sized { // Restores the state the node had right after it was built, resetting its inputs too. + // A parent may also reset a child to rescan it within one execution; only then may a node + // keep what it read (see `ExecContext::restarting`), as a new execution may run in another + // transaction. fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, @@ -574,6 +580,7 @@ pub(crate) struct ExecBuffer { result: ExecResult, table_codec: TableCodec, runtime_probe_stack: Vec, + restarting: bool, } pub(crate) struct ExecArena<'a, 't, T: StorageTransaction + 'a> { @@ -783,6 +790,11 @@ impl<'c, 'a, 't, T: StorageTransaction + 'a> ExecContext<'c, 'a, 't, T> { self.arena.iters.truncate(len); } + #[inline] + pub(crate) fn restarting(&self) -> bool { + self.buffer.restarting + } + #[inline] pub(crate) fn result_tuple(&self) -> &TupleImpl { &self.buffer.result.tuple From e87723fde63da842302339c436c864b1d297bba8 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 4 Nov 2026 11:41:15 +0800 Subject: [PATCH 02/14] perf(storage): borrow strings and binaries from LMDB read-only transactions DataValue::Utf8 and DataValue::Binary now hold Utf8Str and BinaryBytes, which are owned or point into the reads of a MappedScope. An LMDB read-only transaction holds one, so the Char, Varchar and Varbinary columns it decodes point into its memory map instead of being copied. Mapped values never leave a transaction as is: cloning one copies it, and result rows (ResultRow) read without copying but hand values out copied, or lent for 'txn. TransactionIter and ProgramIter borrow their transaction, so their next_tuple lends values until it ends; other iterators lend them only within the callback. The ORM decodes through FromTxnRow and FromTxnValue on those iterators (txn_orm, Transaction::get and Transaction::fetch), and the Model and Projection derives accept one lifetime, so Cow<'txn, str> fields borrow from the transaction. Breaking: the value types of Utf8 and Binary, next_tuple callbacks and FromQueryRow take a ResultRow, and Transaction::fetch returns a TxnOrmIter. --- examples/hello_world.rs | 11 +- examples/transaction.rs | 4 +- kite_sql_serde_macros/src/lib.rs | 1 + kite_sql_serde_macros/src/orm.rs | 80 ++-- kite_sql_serde_macros/src/projection.rs | 71 +-- kite_sql_serde_macros/src/row.rs | 129 +++++ src/binder/expr.rs | 2 +- src/binder/parser.rs | 2 +- src/db.rs | 236 ++++++++- src/db/program.rs | 18 +- src/execution/ddl/mod.rs | 20 +- src/execution/dml/analyze.rs | 2 +- src/execution/dql/aggregate/group_concat.rs | 3 +- src/execution/dql/describe.rs | 17 +- src/execution/dql/explain.rs | 2 +- src/execution/dql/external_sort.rs | 2 +- src/execution/dql/join/nested_loop_join.rs | 2 +- src/execution/dql/show_table.rs | 2 +- src/execution/dql/show_view.rs | 2 +- src/execution/spill/codec.rs | 6 +- src/expression/evaluator.rs | 4 +- src/expression/mod.rs | 2 +- src/function/lower.rs | 2 +- src/function/upper.rs | 2 +- src/macros/mod.rs | 6 +- .../rule/normalization/column_pruning.rs | 2 +- src/orm/dql.rs | 30 +- src/orm/mod.rs | 194 ++++++-- src/python.rs | 4 +- src/serdes/data_value.rs | 13 +- src/storage/lmdb.rs | 151 +++++- src/storage/mod.rs | 91 ++-- src/storage/table_codec.rs | 34 +- src/types/evaluator/binary.rs | 2 +- src/types/evaluator/boolean.rs | 6 +- src/types/evaluator/cast.rs | 66 +-- src/types/evaluator/date.rs | 4 +- src/types/evaluator/datetime.rs | 4 +- src/types/evaluator/decimal.rs | 4 +- src/types/evaluator/float32.rs | 4 +- src/types/evaluator/float64.rs | 4 +- src/types/evaluator/int16.rs | 4 +- src/types/evaluator/int32.rs | 4 +- src/types/evaluator/int64.rs | 4 +- src/types/evaluator/int8.rs | 4 +- src/types/evaluator/time32.rs | 4 +- src/types/evaluator/time64.rs | 4 +- src/types/evaluator/tuple.rs | 2 +- src/types/evaluator/uint16.rs | 4 +- src/types/evaluator/uint32.rs | 4 +- src/types/evaluator/uint64.rs | 4 +- src/types/evaluator/uint8.rs | 4 +- src/types/evaluator/utf8.rs | 26 +- src/types/mapped.rs | 447 ++++++++++++++++++ src/types/mod.rs | 1 + src/types/serialize.rs | 211 ++++++--- src/types/tuple.rs | 100 ++-- src/types/tuple_builder.rs | 6 +- src/types/value.rs | 64 ++- src/wasm.rs | 2 +- tests/macros-test/src/main.rs | 60 ++- tpcc/src/backend/sqlite.rs | 6 +- 62 files changed, 1740 insertions(+), 466 deletions(-) create mode 100644 kite_sql_serde_macros/src/row.rs create mode 100644 src/types/mapped.rs diff --git a/examples/hello_world.rs b/examples/hello_world.rs index 7b2dd302..05bfab8e 100644 --- a/examples/hello_world.rs +++ b/examples/hello_world.rs @@ -18,6 +18,7 @@ mod app { use kite_sql::errors::DatabaseError; use kite_sql::storage::Storage; use kite_sql::Model; + use std::borrow::Cow; use std::env; use std::fs; use std::io::ErrorKind; @@ -41,25 +42,25 @@ mod app { #[derive(Default, Debug, PartialEq, Model)] #[model(table = "my_struct")] - pub struct MyStruct { + pub struct MyStruct<'a> { #[model(primary_key)] pub c1: i32, - pub c2: String, + pub c2: Cow<'a, str>, } fn run_with_database(mut database: Database) -> Result<(), DatabaseError> { database.create_table_if_not_exists::()?; database.insert(&MyStruct { c1: 0, - c2: "zero".to_string(), + c2: "zero".into(), })?; database.insert(&MyStruct { c1: 1, - c2: "one".to_string(), + c2: "one".into(), })?; database.insert(&MyStruct { c1: 2, - c2: "two".to_string(), + c2: "two".into(), })?; database diff --git a/examples/transaction.rs b/examples/transaction.rs index 8e8bdee8..28c32f61 100644 --- a/examples/transaction.rs +++ b/examples/transaction.rs @@ -56,11 +56,11 @@ mod app { let mut iter = database.run("select * from t1")?; iter.next_tuple(|_, tuple| { - assert_eq!(*tuple, vec![DataValue::Int32(0), DataValue::Int32(0)]); + assert_eq!(**tuple, vec![DataValue::Int32(0), DataValue::Int32(0)]); })? .unwrap(); iter.next_tuple(|_, tuple| { - assert_eq!(*tuple, vec![DataValue::Int32(1), DataValue::Int32(1)]); + assert_eq!(**tuple, vec![DataValue::Int32(1), DataValue::Int32(1)]); })? .unwrap(); assert!(iter.next_tuple(|_, _| ())?.is_none()); diff --git a/kite_sql_serde_macros/src/lib.rs b/kite_sql_serde_macros/src/lib.rs index 5444e5ee..29a65654 100644 --- a/kite_sql_serde_macros/src/lib.rs +++ b/kite_sql_serde_macros/src/lib.rs @@ -15,6 +15,7 @@ mod orm; mod projection; mod reference_serialization; +mod row; use proc_macro::TokenStream; use syn::{parse_macro_input, DeriveInput}; diff --git a/kite_sql_serde_macros/src/orm.rs b/kite_sql_serde_macros/src/orm.rs index 1ebe4f3c..e35fd6d7 100644 --- a/kite_sql_serde_macros/src/orm.rs +++ b/kite_sql_serde_macros/src/orm.rs @@ -1,3 +1,4 @@ +use crate::row::{row_impls, single_lifetime, with_static_lifetime}; use darling::ast::Data; use darling::{FromDeriveInput, FromField, FromMeta}; use proc_macro2::{Span, TokenStream}; @@ -58,6 +59,7 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { })?; let table_name_lit = LitStr::new(&table_name, Span::call_site()); let mut generics = orm_opts.generics.clone(); + let lifetime = single_lifetime(&orm_opts.generics, struct_name, "Model")?; let Data::Struct(data_struct) = orm_opts.data else { return Err(Error::new_spanned( @@ -86,6 +88,7 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { Error::new_spanned(struct_name, "Model only supports named struct fields") })?; let field_ty = field.ty; + let static_field_ty = with_static_lifetime(&field_ty, lifetime.as_ref()); if field.skip { if field.primary_key { @@ -182,7 +185,12 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { generics .make_where_clause() .predicates - .push(parse_quote!(#field_ty : ::kite_sql::orm::FromDataValue)); + .push(match &lifetime { + Some(lifetime) => { + parse_quote!(#field_ty : ::kite_sql::orm::FromTxnValue<#lifetime>) + } + None => parse_quote!(#field_ty : ::kite_sql::orm::FromDataValue), + }); generics .make_where_clause() .predicates @@ -216,7 +224,7 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { quote! { ::kite_sql::types::LogicalType::Decimal(Some(#decimal_precision), None) } } } else { - quote! { <#field_ty as ::kite_sql::orm::ModelColumnType>::logical_type() } + quote! { <#static_field_ty as ::kite_sql::orm::ModelColumnType>::logical_type() } }; let default_tokens = if let Some(default_literal) = &default_literal { quote! { Some(::kite_sql::types::value::DataValue::from(#default_literal.to_string())) } @@ -233,12 +241,22 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { __kite_orm_found_fields += 1; } }); - field_initializers.push(quote! { - #field_name: ::kite_sql::orm::take_value_at::<#field_ty>( - tuple, - #field_index_ident, - #column_name_lit, - )? + field_initializers.push(if lifetime.is_some() { + quote! { + #field_name: ::kite_sql::orm::take_txn_value_at::<#field_ty>( + tuple, + #field_index_ident, + #column_name_lit, + )? + } + } else { + quote! { + #field_name: ::kite_sql::orm::take_value_at::<#field_ty>( + tuple, + #field_index_ident, + #column_name_lit, + )? + } }); params.push(quote! { (#column_name_lit, ::kite_sql::orm::ToDataValue::to_data_value(&self.#field_name)) @@ -260,7 +278,7 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { nullable: if #is_primary_key { false } else { - <#field_ty as ::kite_sql::orm::ModelColumnType>::nullable() + <#static_field_ty as ::kite_sql::orm::ModelColumnType>::nullable() }, default, primary_key: #is_primary_key, @@ -379,31 +397,29 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { let primary_key_value = primary_key_value.expect("primary key checked above"); let field_count = field_index_declarations.len(); let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); - - Ok(quote! { - impl #impl_generics ::kite_sql::orm::FromQueryRow - for #struct_name #ty_generics - #where_clause - { - fn from_query_row( - schema: &::kite_sql::types::tuple::SchemaView<'_, '_>, - tuple: &mut ::kite_sql::types::tuple::TupleImpl, - ) -> ::std::result::Result { - let mut __kite_orm_found_fields = 0usize; - #(#field_index_declarations)* - for (__kite_orm_index, __kite_orm_column) in schema.iter().enumerate() { - let __kite_orm_column_name = __kite_orm_column.name(); - #(#field_index_resolvers)* - if __kite_orm_found_fields == #field_count { - break; - } + let row_impls = row_impls( + struct_name, + &generics, + lifetime.as_ref(), + quote! { + let mut __kite_orm_found_fields = 0usize; + #(#field_index_declarations)* + for (__kite_orm_index, __kite_orm_column) in schema.iter().enumerate() { + let __kite_orm_column_name = __kite_orm_column.name(); + #(#field_index_resolvers)* + if __kite_orm_found_fields == #field_count { + break; } - - Ok(Self { - #(#field_initializers),* - }) } - } + + Ok(Self { + #(#field_initializers),* + }) + }, + ); + + Ok(quote! { + #row_impls impl #impl_generics #struct_name #ty_generics #where_clause diff --git a/kite_sql_serde_macros/src/projection.rs b/kite_sql_serde_macros/src/projection.rs index 45fb0312..2c26b352 100644 --- a/kite_sql_serde_macros/src/projection.rs +++ b/kite_sql_serde_macros/src/projection.rs @@ -1,3 +1,4 @@ +use crate::row::{row_impls, single_lifetime}; use darling::ast::Data; use darling::{FromDeriveInput, FromField}; use proc_macro2::{Span, TokenStream}; @@ -33,6 +34,8 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { )); }; + let lifetime = single_lifetime(&projection_opts.generics, struct_name, "Projection")?; + let mut projection_exprs = Vec::new(); let mut field_initializers = Vec::new(); let mut field_index_declarations = Vec::new(); @@ -63,7 +66,12 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { generics .make_where_clause() .predicates - .push(parse_quote!(#field_ty : ::kite_sql::orm::FromDataValue)); + .push(match &lifetime { + Some(lifetime) => { + parse_quote!(#field_ty : ::kite_sql::orm::FromTxnValue<#lifetime>) + } + None => parse_quote!(#field_ty : ::kite_sql::orm::FromDataValue), + }); projection_exprs.push(if rename.is_some() { quote! { @@ -86,17 +94,43 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { __kite_orm_found_fields += 1; } }); - field_initializers.push(quote! { - #field_name: ::kite_sql::orm::take_value_at::<#field_ty>( - tuple, - #field_index_ident, - #field_name_lit, - )? + field_initializers.push(if lifetime.is_some() { + quote! { + #field_name: ::kite_sql::orm::take_txn_value_at::<#field_ty>( + tuple, + #field_index_ident, + #field_name_lit, + )? + } + } else { + quote! { + #field_name: ::kite_sql::orm::take_value_at::<#field_ty>( + tuple, + #field_index_ident, + #field_name_lit, + )? + } }); } let field_count = field_index_declarations.len(); let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + let decode = quote! { + let mut __kite_orm_found_fields = 0usize; + #(#field_index_declarations)* + for (__kite_orm_index, __kite_orm_column) in schema.iter().enumerate() { + let __kite_orm_column_name = __kite_orm_column.name(); + #(#field_index_resolvers)* + if __kite_orm_found_fields == #field_count { + break; + } + } + + Ok(Self { + #(#field_initializers),* + }) + }; + let row_impls = row_impls(struct_name, &generics, lifetime.as_ref(), decode); Ok(quote! { impl #impl_generics ::kite_sql::orm::Projection for #struct_name #ty_generics @@ -116,27 +150,6 @@ pub(crate) fn handle(ast: DeriveInput) -> Result { } } - impl #impl_generics ::kite_sql::orm::FromQueryRow for #struct_name #ty_generics - #where_clause - { - fn from_query_row( - schema: &::kite_sql::types::tuple::SchemaView<'_, '_>, - tuple: &mut ::kite_sql::types::tuple::TupleImpl, - ) -> ::std::result::Result { - let mut __kite_orm_found_fields = 0usize; - #(#field_index_declarations)* - for (__kite_orm_index, __kite_orm_column) in schema.iter().enumerate() { - let __kite_orm_column_name = __kite_orm_column.name(); - #(#field_index_resolvers)* - if __kite_orm_found_fields == #field_count { - break; - } - } - - Ok(Self { - #(#field_initializers),* - }) - } - } + #row_impls }) } diff --git a/kite_sql_serde_macros/src/row.rs b/kite_sql_serde_macros/src/row.rs new file mode 100644 index 00000000..bc85f0cf --- /dev/null +++ b/kite_sql_serde_macros/src/row.rs @@ -0,0 +1,129 @@ +use proc_macro2::{Spacing, TokenStream, TokenTree}; +use quote::{quote, ToTokens}; +use syn::{parse_quote, Error, Generics, Ident, Lifetime, Type}; + +/// The single lifetime of a derived struct, which its fields may borrow from a transaction. +pub(crate) fn single_lifetime( + generics: &Generics, + struct_name: &Ident, + derive_name: &str, +) -> Result, Error> { + let mut lifetimes = generics.lifetimes(); + let lifetime = lifetimes.next().map(|param| param.lifetime.clone()); + if lifetimes.next().is_some() { + return Err(Error::new_spanned( + struct_name, + format!("{derive_name} supports at most one lifetime parameter"), + )); + } + Ok(lifetime) +} + +/// Implements `FromQueryRow` and `FromTxnRow` with `decode`, which reads `schema` and `tuple`. +/// +/// A struct with a lifetime borrows from the transaction when it decodes from a transaction's +/// result row, and owns copies of every value when it decodes from any result row. +pub(crate) fn row_impls( + struct_name: &Ident, + generics: &Generics, + lifetime: Option<&Lifetime>, + decode: TokenStream, +) -> TokenStream { + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + match lifetime { + Some(lifetime) => quote! { + impl #impl_generics ::kite_sql::orm::FromTxnRow<#lifetime> for #struct_name #ty_generics + #where_clause + { + fn from_txn_row( + schema: &::kite_sql::types::tuple::SchemaView<'_, '_>, + tuple: &mut ::kite_sql::db::ResultRow<'_, #lifetime>, + ) -> ::std::result::Result { + #decode + } + } + + impl #impl_generics ::kite_sql::orm::FromQueryRow for #struct_name #ty_generics + #where_clause + { + fn from_query_row( + schema: &::kite_sql::types::tuple::SchemaView<'_, '_>, + tuple: &mut ::kite_sql::db::ResultRow<'_, '_>, + ) -> ::std::result::Result { + >::from_txn_row( + schema, + &mut ::kite_sql::db::ResultRow::materialized(tuple), + ) + } + } + }, + None => { + let mut txn_generics = generics.clone(); + txn_generics.params.insert(0, parse_quote!('__kite_txn)); + let (txn_impl_generics, _, _) = txn_generics.split_for_impl(); + quote! { + impl #impl_generics ::kite_sql::orm::FromQueryRow for #struct_name #ty_generics + #where_clause + { + fn from_query_row( + schema: &::kite_sql::types::tuple::SchemaView<'_, '_>, + tuple: &mut ::kite_sql::db::ResultRow<'_, '_>, + ) -> ::std::result::Result { + #decode + } + } + + impl #txn_impl_generics ::kite_sql::orm::FromTxnRow<'__kite_txn> + for #struct_name #ty_generics + #where_clause + { + fn from_txn_row( + schema: &::kite_sql::types::tuple::SchemaView<'_, '_>, + tuple: &mut ::kite_sql::db::ResultRow<'_, '__kite_txn>, + ) -> ::std::result::Result { + ::from_query_row(schema, tuple) + } + } + } + } + } +} + +/// `ty` with `lifetime` replaced by `'static`, for statics, which cannot name the struct's +/// lifetime. Lifetimes never affect what a column type maps to. +pub(crate) fn with_static_lifetime(ty: &Type, lifetime: Option<&Lifetime>) -> TokenStream { + match lifetime { + Some(lifetime) => replace_lifetime(ty.to_token_stream(), &lifetime.ident), + None => ty.to_token_stream(), + } +} + +fn replace_lifetime(tokens: TokenStream, name: &Ident) -> TokenStream { + let mut output = Vec::new(); + let mut tokens = tokens.into_iter().peekable(); + while let Some(token) = tokens.next() { + match token { + TokenTree::Punct(punct) + if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => + { + match tokens.peek() { + Some(TokenTree::Ident(ident)) if ident == name => { + tokens.next(); + output.extend(quote!('static)); + } + _ => output.push(TokenTree::Punct(punct)), + } + } + TokenTree::Group(group) => { + let mut replaced = proc_macro2::Group::new( + group.delimiter(), + replace_lifetime(group.stream(), name), + ); + replaced.set_span(group.span()); + output.push(TokenTree::Group(replaced)); + } + token => output.push(token), + } + } + output.into_iter().collect() +} diff --git a/src/binder/expr.rs b/src/binder/expr.rs index af0b6082..c1b26c80 100644 --- a/src/binder/expr.rs +++ b/src/binder/expr.rs @@ -693,7 +693,7 @@ impl<'a, T: StorageTransaction, A: AsRef<[(usize, LogicalType)]>> Binder<'a, '_, pub(crate) fn wildcard_expr() -> ScalarExpression { ScalarExpression::Constant(DataValue::Utf8 { - value: "*".to_string(), + value: "*".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }) diff --git a/src/binder/parser.rs b/src/binder/parser.rs index 538ff3df..9aac5ad4 100644 --- a/src/binder/parser.rs +++ b/src/binder/parser.rs @@ -2344,7 +2344,7 @@ impl<'a, 'parent, T: StorageTransaction, A: AsRef<[(usize, LogicalType)]>> DatabaseError::InvalidValue("typed string literal must be a string".to_string()) })?; let value = DataValue::Utf8 { - value: raw, + value: raw.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } diff --git a/src/db.rs b/src/db.rs index 85f5caa0..49e09010 100644 --- a/src/db.rs +++ b/src/db.rs @@ -41,7 +41,7 @@ use crate::optimizer::heuristic::optimizer::HepOptimizerPipeline; use crate::optimizer::rule::implementation::ImplementationRuleImpl; use crate::optimizer::rule::normalization::NormalizationRuleImpl; #[cfg(feature = "orm")] -use crate::orm::FromQueryRow; +use crate::orm::{FromQueryRow, FromTxnRow}; use crate::planner::operator::Operator; use crate::planner::{ExecMetaArena, LogicalPlan, MetaArena, PlanArena, TableArenaCell}; #[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))] @@ -59,9 +59,11 @@ use crate::types::value::DataValue; use crate::types::LogicalType; use ordered_float::OrderedFloat; pub use program::{Program, ProgramIter}; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::mem; +use std::ops::Deref; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -1190,6 +1192,140 @@ where } } +/// A result row, lent to the callback of an iterator's `next_tuple`. +/// +/// It reads as a [`TupleImpl`] without copying. Its strings and binaries may point into the +/// transaction's reads, so values leave the row copied out of those through [`ResultRow::take`], +/// [`ResultRow::take_values`], [`ResultRow::take_value`] or `clone`, or lent for `'txn` through +/// [`ResultRow::take_txn_value`]. `'txn` lasts as long as [`TransactionIter`] or `ProgramIter` +/// borrows its transaction, and only the callback for [`DatabaseIter`] or through +/// [`ResultIter`]. +#[derive(Debug)] +pub struct ResultRow<'r, 'txn> { + tuple: &'r mut TupleImpl, + _txn: PhantomData<&'txn ()>, +} + +impl<'r, 'txn> ResultRow<'r, 'txn> { + pub fn new(tuple: &'r mut TupleImpl) -> Self { + Self { + tuple, + _txn: PhantomData, + } + } + + /// Lends `row` after copying its values out of the transaction's reads, so nothing it hands + /// out borrows the transaction, whatever `'txn` is. + #[doc(hidden)] + pub fn materialized(row: &'r mut ResultRow<'_, '_>) -> Self { + row.tuple.materialize(); + Self::new(&mut *row.tuple) + } + + /// Takes the whole row, leaving it empty. + pub fn take(&mut self) -> TupleImpl { + let mut tuple = mem::take(self.tuple); + tuple.materialize(); + tuple + } + + /// Takes every value of the row, leaving it empty. + pub fn take_values(&mut self) -> Vec { + let mut values = self.tuple.take_values(); + values.iter_mut().for_each(DataValue::materialize); + values + } + + /// Takes the value at `index`, leaving NULL in its place. + pub fn take_value(&mut self, index: usize) -> Option { + let mut value = mem::replace(self.tuple.get_mut(index)?, DataValue::Null); + value.materialize(); + Some(value) + } + + /// Takes the value at `index` without copying its strings and binaries, leaving NULL in its + /// place. + pub fn take_txn_value(&mut self, index: usize) -> Option> { + let value = mem::replace(self.tuple.get_mut(index)?, DataValue::Null); + Some(TxnValue { + value, + _txn: PhantomData, + }) + } +} + +impl Deref for ResultRow<'_, '_> { + type Target = TupleImpl; + + fn deref(&self) -> &TupleImpl { + self.tuple + } +} + +/// A value taken from a [`ResultRow`] without copying. Its strings and binaries may still point +/// into the transaction's reads, which stay valid for `'txn`. +#[derive(Debug)] +pub struct TxnValue<'txn> { + value: DataValue, + _txn: PhantomData<&'txn ()>, +} + +impl<'txn> TxnValue<'txn> { + pub fn is_null(&self) -> bool { + self.value.is_null() + } + + pub fn cast(self, ty: &LogicalType) -> Result { + Ok(Self { + value: self.value.cast(ty)?, + _txn: PhantomData, + }) + } + + pub fn into_str(self) -> Result, DataValue> { + match self.value { + // SAFETY: a mapped value comes from the transaction borrowed for `'txn`, which cannot + // end before `'txn` does. + DataValue::Utf8 { value, .. } => Ok(unsafe { value.into_cow() }), + value => Err(value), + } + } + + pub fn into_bytes(self) -> Result, DataValue> { + match self.value { + // SAFETY: as in `into_str`. + DataValue::Binary { value, .. } => Ok(unsafe { value.into_cow() }), + value => Err(value), + } + } + + pub fn into_value(self) -> DataValue { + let mut value = self.value; + value.materialize(); + value + } +} + +/// A result iterator that borrows its transaction for `'txn`, such as [`TransactionIter`], so its +/// rows lend values for as long. +pub trait TxnResultIter<'txn>: ResultIter { + /// Like [`ResultIter::next_tuple`], but the row lends values for `'txn`. The iterators' own + /// `next_tuple` does the same; this is for code generic over them. + fn next_txn_tuple( + &mut self, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'txn>) -> R, + ) -> Result, DatabaseError>; + + #[cfg(feature = "orm")] + fn txn_orm(self) -> TxnOrmIter<'txn, Self, T> + where + Self: Sized, + T: FromTxnRow<'txn>, + { + TxnOrmIter::new(self) + } +} + /// Common interface for result iterators returned by database execution APIs. pub trait ResultIter { /// Borrows the output schema for the current result set. @@ -1198,7 +1334,7 @@ pub trait ResultIter { /// Advances to the next row and runs `f` while the executor row slot is borrowed. fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut TupleImpl) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, ) -> Result, DatabaseError>; /// Finishes consuming the iterator and flushes any remaining work. @@ -1266,6 +1402,46 @@ where } } +#[cfg(feature = "orm")] +pub struct TxnOrmIter<'txn, I, T> { + inner: I, + _marker: PhantomData<(&'txn (), T)>, +} + +#[cfg(feature = "orm")] +impl<'txn, I, T> TxnOrmIter<'txn, I, T> +where + I: TxnResultIter<'txn>, + T: FromTxnRow<'txn>, +{ + fn new(inner: I) -> Self { + Self { + inner, + _marker: PhantomData, + } + } + + pub fn done(self) -> Result<(), DatabaseError> { + self.inner.done() + } +} + +#[cfg(feature = "orm")] +impl<'txn, I, T> Iterator for TxnOrmIter<'txn, I, T> +where + I: TxnResultIter<'txn>, + T: FromTxnRow<'txn>, +{ + type Item = Result; + + fn next(&mut self) -> Option { + self.inner + .next_txn_tuple(|schema, row| T::from_txn_row(schema, row)) + .transpose() + .map(|row| row.and_then(std::convert::identity)) + } +} + /// Raw result iterator returned by database execution APIs. pub struct DatabaseIter<'a, S: Storage + 'a> { pub(crate) transaction: *mut S::TransactionType<'a>, @@ -1292,7 +1468,7 @@ impl<'a, S: Storage> DatabaseIter<'a, S> { #[inline] pub fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut TupleImpl) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, ) -> Result, DatabaseError> { unsafe { (*self.inner).next_tuple(f) } } @@ -1316,7 +1492,7 @@ impl ResultIter for DatabaseIter<'_, S> { fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut TupleImpl) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, ) -> Result, DatabaseError> { DatabaseIter::next_tuple(self, f) } @@ -1386,26 +1562,26 @@ struct TransactionGuard<'a, 'p, T: StorageTransaction + 'p> { impl TransactionGuard<'_, '_, T> { #[inline] - fn finish(&mut self, ddl_apply: &mut Vec) { + fn finish(&mut self) { if !self.active { return; } self.active = false; self.arena.release_iters(); - ddl_apply.extend(self.arena.take_ddl_apply()); } } impl<'p, T: StorageTransaction + 'p> TransactionGuard<'_, 'p, T> { /// Pulls the next row of `executor`, finishing the execution once the rows run out /// or the execution fails. + /// + /// The row lends values for `'txn`, which the caller must not let outlive the transaction. #[inline] - fn next_tuple( + fn next_tuple<'txn, A: MetaArena + 'p, R>( &mut self, executor: &mut Executor<'p>, plan_arena: &mut ExecMetaArena, - ddl_apply: &mut Vec, - f: impl FnOnce(&SchemaView<'_, '_>, &mut TupleImpl) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'txn>) -> R, ) -> Result, DatabaseError> { if !self.active { return Ok(None); @@ -1413,14 +1589,17 @@ impl<'p, T: StorageTransaction + 'p> TransactionGuard<'_, 'p, T> { match executor.next_tuple(&mut self.arena, plan_arena) { Ok(true) => { let schema = SchemaView::new(&executor.schema, plan_arena); - Ok(Some(f(&schema, executor.buffer.result_mut()))) + Ok(Some(f( + &schema, + &mut ResultRow::new(executor.buffer.result_mut()), + ))) } Ok(false) => { - self.finish(ddl_apply); + self.finish(); Ok(None) } Err(err) => { - self.finish(ddl_apply); + self.finish(); Err(err) } } @@ -1429,7 +1608,7 @@ impl<'p, T: StorageTransaction + 'p> TransactionGuard<'_, 'p, T> { impl Drop for TransactionGuard<'_, '_, T> { fn drop(&mut self) { - self.finish(&mut Vec::new()); + self.finish(); } } @@ -1442,7 +1621,6 @@ pub struct TransactionIter<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p guard: TransactionGuard<'a, 'p, T>, executor: Executor<'p>, plan_arena: ExecMetaArena, - ddl_apply: Vec, } impl<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p> TransactionIter<'a, 'p, T, A> { @@ -1457,7 +1635,6 @@ impl<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p> TransactionIter<'a, }, executor, plan_arena, - ddl_apply: Vec::new(), } } @@ -1466,17 +1643,15 @@ impl<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p> TransactionIter<'a, f(&SchemaView::new(&self.executor.schema, &self.plan_arena)) } + /// Advances to the next row and runs `f` on it. The iterator borrows its transaction for + /// `'a`, so the row lends values for as long (see [`ResultRow::take_txn_value`]). #[inline] pub fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut TupleImpl) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R, ) -> Result, DatabaseError> { - self.guard.next_tuple( - &mut self.executor, - &mut self.plan_arena, - &mut self.ddl_apply, - f, - ) + self.guard + .next_tuple(&mut self.executor, &mut self.plan_arena, f) } #[inline] @@ -1490,11 +1665,11 @@ impl<'a, T: StorageTransaction + 'a> TransactionIter<'a, 'a, T, PlanArena<'a>> { fn done_with_ddl_apply(mut self) -> Result<(PlanArena<'a>, Vec), DatabaseError> { while self.next_tuple(|_, _| ())?.is_some() {} let Self { - guard, + mut guard, plan_arena, - ddl_apply, .. } = self; + let ddl_apply = guard.arena.take_ddl_apply(); drop(guard); Ok((plan_arena.into_parent(), ddl_apply)) } @@ -1509,7 +1684,7 @@ impl<'p, T: StorageTransaction + 'p, A: MetaArena + 'p> ResultIter fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut TupleImpl) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, ) -> Result, DatabaseError> { TransactionIter::next_tuple(self, f) } @@ -1519,6 +1694,17 @@ impl<'p, T: StorageTransaction + 'p, A: MetaArena + 'p> ResultIter } } +impl<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p> TxnResultIter<'a> + for TransactionIter<'a, 'p, T, A> +{ + fn next_txn_tuple( + &mut self, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R, + ) -> Result, DatabaseError> { + TransactionIter::next_tuple(self, f) + } +} + // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] pub(crate) mod test { diff --git a/src/db/program.rs b/src/db/program.rs index 99c14fd1..2c3cab10 100644 --- a/src/db/program.rs +++ b/src/db/program.rs @@ -171,7 +171,7 @@ pub struct ProgramIter<'a, 'p, T: StorageTransaction + 'p> { program: &'a mut Program<'p>, } -impl<'p, T: StorageTransaction + 'p> ProgramIter<'_, 'p, T> { +impl<'a, 'p, T: StorageTransaction + 'p> ProgramIter<'a, 'p, T> { #[inline] pub fn schema(&self, f: impl FnOnce(&SchemaView<'_, '_>) -> R) -> R { let Program { executor, arena } = &*self.program; @@ -181,11 +181,10 @@ impl<'p, T: StorageTransaction + 'p> ProgramIter<'_, 'p, T> { #[inline] pub fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut TupleImpl) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R, ) -> Result, DatabaseError> { let Program { executor, arena } = &mut *self.program; - // A program never applies DDL. - self.guard.next_tuple(executor, arena, &mut Vec::new(), f) + self.guard.next_tuple(executor, arena, f) } #[inline] @@ -195,6 +194,15 @@ impl<'p, T: StorageTransaction + 'p> ProgramIter<'_, 'p, T> { } } +impl<'a, 'p, T: StorageTransaction + 'p> TxnResultIter<'a> for ProgramIter<'a, 'p, T> { + fn next_txn_tuple( + &mut self, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R, + ) -> Result, DatabaseError> { + ProgramIter::next_tuple(self, f) + } +} + impl<'p, T: StorageTransaction + 'p> ResultIter for ProgramIter<'_, 'p, T> { fn schema(&self, f: impl FnOnce(&SchemaView<'_, '_>) -> R) -> R { ProgramIter::schema(self, f) @@ -202,7 +210,7 @@ impl<'p, T: StorageTransaction + 'p> ResultIter for ProgramIter<'_, 'p, T> { fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut TupleImpl) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, ) -> Result, DatabaseError> { ProgramIter::next_tuple(self, f) } diff --git a/src/execution/ddl/mod.rs b/src/execution/ddl/mod.rs index f7d19561..8754ce2d 100644 --- a/src/execution/ddl/mod.rs +++ b/src/execution/ddl/mod.rs @@ -78,14 +78,18 @@ fn read_tuple_batch( .last_mut() .expect("batch contains the tuple that was just pushed") }; - TableCodec::decode_tuple_into( - tuple.output_mut(), - old_deserializers, - Some(tuple_id), - value, - old_deserializers.len(), - pool, - )?; + // SAFETY: without a scope every value is copied. + unsafe { + TableCodec::decode_tuple_into( + tuple.output_mut(), + old_deserializers, + Some(tuple_id), + value, + old_deserializers.len(), + pool, + None, + )? + }; len += 1; } diff --git a/src/execution/dml/analyze.rs b/src/execution/dml/analyze.rs index 6651af85..d3d2a293 100644 --- a/src/execution/dml/analyze.rs +++ b/src/execution/dml/analyze.rs @@ -228,7 +228,7 @@ impl<'a> Analyze<'a> { meta, }); values.push(DataValue::Utf8 { - value: format!("{table_name}/{index_id}"), + value: format!("{table_name}/{index_id}").into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); diff --git a/src/execution/dql/aggregate/group_concat.rs b/src/execution/dql/aggregate/group_concat.rs index 778e56a6..ff77457c 100644 --- a/src/execution/dql/aggregate/group_concat.rs +++ b/src/execution/dql/aggregate/group_concat.rs @@ -47,12 +47,13 @@ impl Accumulator for GroupConcatAccumulator { } match &mut self.result { DataValue::Utf8 { value: text, .. } => { + let text = text.to_mut(); text.push(','); let _ = write!(text, "{value}"); } result => { *result = DataValue::Utf8 { - value: value.to_string(), + value: value.to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } diff --git a/src/execution/dql/describe.rs b/src/execution/dql/describe.rs index a8e1f142..4b636b51 100644 --- a/src/execution/dql/describe.rs +++ b/src/execution/dql/describe.rs @@ -23,19 +23,19 @@ use crate::types::CharLengthUnits; use std::sync::LazyLock; static PRIMARY_KEY_TYPE: LazyLock = LazyLock::new(|| DataValue::Utf8 { - value: String::from("PRIMARY"), + value: "PRIMARY".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); static UNIQUE_KEY_TYPE: LazyLock = LazyLock::new(|| DataValue::Utf8 { - value: String::from("UNIQUE"), + value: "UNIQUE".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); static EMPTY_KEY_TYPE: LazyLock = LazyLock::new(|| DataValue::Utf8 { - value: String::from("EMPTY"), + value: "EMPTY".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); @@ -130,12 +130,12 @@ fn fill_describe_row(values: &mut Vec, column: &ColumnCatalog, defaul let datatype = column.datatype(); values.push(DataValue::Utf8 { - value: column.name().to_string(), + value: column.name().to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); values.push(DataValue::Utf8 { - value: datatype.to_string(), + value: datatype.to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); @@ -143,18 +143,19 @@ fn fill_describe_row(values: &mut Vec, column: &ColumnCatalog, defaul value: datatype .raw_len() .map(|len| len.to_string()) - .unwrap_or_else(|| "variable".to_string()), + .unwrap_or_else(|| "variable".to_string()) + .into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); values.push(DataValue::Utf8 { - value: column.nullable().to_string(), + value: column.nullable().to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); values.push(key_value(column)); values.push(DataValue::Utf8 { - value: default, + value: default.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); diff --git a/src/execution/dql/explain.rs b/src/execution/dql/explain.rs index 4a8e7eb8..68fa0ff7 100644 --- a/src/execution/dql/explain.rs +++ b/src/execution/dql/explain.rs @@ -65,7 +65,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Explain<'a> { output.pk = None; output.values.clear(); output.values.push(DataValue::Utf8 { - value: plan, + value: plan.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); diff --git a/src/execution/dql/external_sort.rs b/src/execution/dql/external_sort.rs index fd7cb86b..ffc2de11 100644 --- a/src/execution/dql/external_sort.rs +++ b/src/execution/dql/external_sort.rs @@ -384,7 +384,7 @@ mod test { plan_arena.set_init_value( reference, DataValue::Utf8 { - value: format!("row-{sequence}"), + value: format!("row-{sequence}").into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }, diff --git a/src/execution/dql/join/nested_loop_join.rs b/src/execution/dql/join/nested_loop_join.rs index a03ce36e..c097d432 100644 --- a/src/execution/dql/join/nested_loop_join.rs +++ b/src/execution/dql/join/nested_loop_join.rs @@ -386,7 +386,7 @@ mod test { .iter() .map(|value| match value { DataValue::Null => None, - DataValue::Utf8 { value, .. } => Some(value.clone()), + DataValue::Utf8 { value, .. } => Some(value.to_string()), other => Some(other.to_string()), }) .collect() diff --git a/src/execution/dql/show_table.rs b/src/execution/dql/show_table.rs index b7072ab2..5744d935 100644 --- a/src/execution/dql/show_table.rs +++ b/src/execution/dql/show_table.rs @@ -70,7 +70,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ShowTables { output.pk = None; output.values.clear(); output.values.push(DataValue::Utf8 { - value: table.table_name.to_string(), + value: table.table_name.to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); diff --git a/src/execution/dql/show_view.rs b/src/execution/dql/show_view.rs index 90751b41..0a74642b 100644 --- a/src/execution/dql/show_view.rs +++ b/src/execution/dql/show_view.rs @@ -70,7 +70,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ShowViews { output.pk = None; output.values.clear(); output.values.push(DataValue::Utf8 { - value: name.to_string(), + value: name.to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); diff --git a/src/execution/spill/codec.rs b/src/execution/spill/codec.rs index dce65171..0542e85a 100644 --- a/src/execution/spill/codec.rs +++ b/src/execution/spill/codec.rs @@ -374,8 +374,8 @@ fn seq_encoded_len<'v, T: SpillCodec + 'v>( fn estimated_dynamic_value_size(value: &DataValue) -> usize { match value { - DataValue::Utf8 { value, .. } => value.capacity(), - DataValue::Binary { value, .. } => value.capacity(), + DataValue::Utf8 { value, .. } => value.heap_size(), + DataValue::Binary { value, .. } => value.heap_size(), DataValue::Tuple(values) => values .capacity() .saturating_mul(size_of::()) @@ -411,7 +411,7 @@ mod tests { fn values() -> Vec { let values = vec![ DataValue::Binary { - value: vec![0, 255, 128], + value: vec![0, 255, 128].into(), len: Some(8), }, DataValue::from(Vec::::new()), diff --git a/src/expression/evaluator.rs b/src/expression/evaluator.rs index a3cb6c94..ba10d710 100644 --- a/src/expression/evaluator.rs +++ b/src/expression/evaluator.rs @@ -217,7 +217,7 @@ impl ScalarExpression { } Ok(Cow::Owned(DataValue::Utf8 { - value: string, + value: string.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, })) @@ -259,7 +259,7 @@ impl ScalarExpression { let string_trimmed = trim_string(string, &trim_what, *trim_where); Ok(Cow::Owned(DataValue::Utf8 { - value: string_trimmed, + value: string_trimmed.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, })) diff --git a/src/expression/mod.rs b/src/expression/mod.rs index a39c1750..49e39a5c 100644 --- a/src/expression/mod.rs +++ b/src/expression/mod.rs @@ -1194,7 +1194,7 @@ mod test { fn_assert( &mut cursor, ScalarExpression::Constant(DataValue::Utf8 { - value: "hello".to_string(), + value: "hello".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }), diff --git a/src/function/lower.rs b/src/function/lower.rs index a4fc59b8..77026511 100644 --- a/src/function/lower.rs +++ b/src/function/lower.rs @@ -60,7 +60,7 @@ impl ScalarFunctionImpl for Lower { } Ok(match value.as_ref() { DataValue::Utf8 { value, ty, unit } => DataValue::Utf8 { - value: value.to_lowercase(), + value: value.to_lowercase().into(), ty: ty.clone(), unit: *unit, }, diff --git a/src/function/upper.rs b/src/function/upper.rs index b121d83b..739322e0 100644 --- a/src/function/upper.rs +++ b/src/function/upper.rs @@ -60,7 +60,7 @@ impl ScalarFunctionImpl for Upper { } Ok(match value.as_ref() { DataValue::Utf8 { value, ty, unit } => DataValue::Utf8 { - value: value.to_uppercase(), + value: value.to_uppercase().into(), ty: ty.clone(), unit: *unit, }, diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 71b7ddbc..ec32dbc3 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -41,13 +41,13 @@ macro_rules! from_tuple { impl ::kite_sql::orm::FromQueryRow for $struct_name { fn from_query_row( schema: &::kite_sql::types::tuple::SchemaView<'_, '_>, - tuple: &mut ::kite_sql::types::tuple::TupleImpl, + tuple: &mut ::kite_sql::db::ResultRow<'_, '_>, ) -> ::std::result::Result { - fn try_get(tuple: &mut ::kite_sql::types::tuple::TupleImpl, schema: &::kite_sql::types::tuple::SchemaView<'_, '_>, field_name: &str) -> Option<::kite_sql::types::value::DataValue> { + fn try_get(tuple: &mut ::kite_sql::db::ResultRow<'_, '_>, schema: &::kite_sql::types::tuple::SchemaView<'_, '_>, field_name: &str) -> Option<::kite_sql::types::value::DataValue> { let ty = ::kite_sql::types::LogicalType::type_trans::()?; let idx = schema.position(field_name)?; - std::mem::replace(&mut tuple[idx], ::kite_sql::types::value::DataValue::Null).cast(&ty).ok() + tuple.take_value(idx)?.cast(&ty).ok() } let mut struct_instance = $struct_name::default(); diff --git a/src/optimizer/rule/normalization/column_pruning.rs b/src/optimizer/rule/normalization/column_pruning.rs index 7fd84e3f..37d7d148 100644 --- a/src/optimizer/rule/normalization/column_pruning.rs +++ b/src/optimizer/rule/normalization/column_pruning.rs @@ -376,7 +376,7 @@ impl ColumnPruning { if op.agg_calls.is_empty() && op.groupby_exprs.is_empty() { let value = DataValue::Utf8 { - value: "*".to_string(), + value: "*".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }; diff --git a/src/orm/dql.rs b/src/orm/dql.rs index 99ce6ea5..b63c8ba8 100644 --- a/src/orm/dql.rs +++ b/src/orm/dql.rs @@ -119,7 +119,7 @@ impl Database { /// assert_eq!(users[0].name, "Alice"); /// ``` pub fn fetch(&self) -> Result, M>, DatabaseError> { - orm_list::<_, M>(self) + Ok(bind_orm_context(self, true, |ctx| ctx.from::()?.finish())?.orm::()) } /// Lists all table names. @@ -249,16 +249,28 @@ impl<'a, S: Storage> Transaction<'a, S> { explain_orm_context(self, build) } - /// Loads a single model by primary key inside the current transaction. - pub fn get(&mut self, key: &M::PrimaryKey) -> Result, DatabaseError> { - orm_get::<_, M>(self, key) + /// Loads a single model by primary key inside the current transaction. A model with a + /// lifetime may borrow its strings and binaries from the transaction. + pub fn get<'t, M: Model + FromTxnRow<'t>>( + &'t mut self, + key: &M::PrimaryKey, + ) -> Result, DatabaseError> { + let mut iter = bind_get::<_, M>(self, key)?; + Ok( + match iter.next_tuple(|schema, row| M::from_txn_row(schema, row))? { + Some(row) => Some(row?), + None => None, + }, + ) } - /// Fetches all rows for a model inside the current transaction. - pub fn fetch( - &mut self, - ) -> Result>, M>, DatabaseError> { - orm_list::<_, M>(self) + /// Fetches all rows for a model inside the current transaction. A model with a lifetime may + /// borrow its strings and binaries from the transaction. + pub fn fetch<'t, M: Model + FromTxnRow<'t>>( + &'t mut self, + ) -> Result>, M>, DatabaseError> + { + Ok(bind_orm_context(self, true, |ctx| ctx.from::()?.finish())?.txn_orm::()) } /// Lists all table names inside the current transaction. diff --git a/src/orm/mod.rs b/src/orm/mod.rs index 93e1e693..1ad7e967 100644 --- a/src/orm/mod.rs +++ b/src/orm/mod.rs @@ -6,7 +6,8 @@ use crate::binder::{ }; use crate::catalog::{ColumnCatalog, ColumnRef, TableCatalog}; use crate::db::{ - BindSource, Database, DatabaseIter, Executor, OrmIter, ResultIter, Transaction, TransactionIter, + BindSource, Database, DatabaseIter, Executor, OrmIter, ResultIter, ResultRow, Transaction, + TransactionIter, TxnOrmIter, TxnResultIter, TxnValue, }; use crate::errors::DatabaseError; pub use crate::expression::agg::AggKind; @@ -18,7 +19,7 @@ use crate::planner::operator::mark_apply::MarkApplyQuantifier; use crate::planner::operator::sort::SortField; use crate::planner::{ExprRef, LogicalPlan, PlanArena}; use crate::storage::{Storage, StorageTransaction}; -use crate::types::tuple::{SchemaView, TupleImpl}; +use crate::types::tuple::SchemaView; use crate::types::value::DataValue; use crate::types::CharLengthUnits; use crate::types::LogicalType; @@ -85,7 +86,7 @@ pub struct DescribeColumn { impl FromQueryRow for DescribeColumn { fn from_query_row( _: &SchemaView<'_, '_>, - tuple: &mut TupleImpl, + tuple: &mut ResultRow<'_, '_>, ) -> Result { let field = describe_text_value(take_projected_value(tuple, 0)); let data_type = describe_text_value(take_projected_value(tuple, 1)); @@ -2575,7 +2576,7 @@ where } #[doc(hidden)] -pub trait Projection: FromQueryRow { +pub trait Projection { fn bind_projection<'ctx, 'bind, 'parent, 'arena, T, A>( scope: &mut ExprBindScope<'ctx, 'bind, 'parent, 'arena, T, A>, relation: &str, @@ -2710,7 +2711,7 @@ where fn describe_text_value(value: Option) -> String { match value { - Some(DataValue::Utf8 { value, .. }) => value, + Some(DataValue::Utf8 { value, .. }) => value.into_string(), Some(other) => other.to_string(), None => String::new(), } @@ -2790,7 +2791,7 @@ pub trait FromDataValue: Sized { /// ``` pub trait FromQueryTuple: Sized { /// Decodes one projected tuple into `Self`. - fn from_query_tuple(tuple: &mut TupleImpl) -> Result; + fn from_query_tuple(tuple: &mut ResultRow<'_, '_>) -> Result; } /// Conversion trait from a query result row into a Rust value. @@ -2800,10 +2801,65 @@ pub trait FromQueryRow: Sized { /// Decodes one result row into `Self`. fn from_query_row( schema: &SchemaView<'_, '_>, - tuple: &mut TupleImpl, + tuple: &mut ResultRow<'_, '_>, ) -> Result; } +pub trait FromTxnRow<'txn>: Sized { + fn from_txn_row( + schema: &SchemaView<'_, '_>, + row: &mut ResultRow<'_, 'txn>, + ) -> Result; +} + +pub trait FromTxnValue<'txn>: Sized { + fn logical_type() -> Option; + + fn from_txn_value(value: TxnValue<'txn>) -> Result; +} + +impl<'txn, T: FromDataValue> FromTxnValue<'txn> for T { + fn logical_type() -> Option { + T::logical_type() + } + + fn from_txn_value(value: TxnValue<'txn>) -> Result { + T::from_data_value(value.into_value()) + } +} + +macro_rules! impl_from_txn_value_for_cow { + ($borrowed:ty, $owned:ty, $into:ident) => { + impl<'txn> FromTxnValue<'txn> for Cow<'txn, $borrowed> { + fn logical_type() -> Option { + <$owned as FromDataValue>::logical_type() + } + + fn from_txn_value(value: TxnValue<'txn>) -> Result { + value + .$into() + .map_err(|value| invalid_from_data_value::(&value)) + } + } + + impl<'txn> FromTxnValue<'txn> for Option> { + fn logical_type() -> Option { + <$owned as FromDataValue>::logical_type() + } + + fn from_txn_value(value: TxnValue<'txn>) -> Result { + if value.is_null() { + return Ok(None); + } + Cow::from_txn_value(value).map(Some) + } + } + }; +} + +impl_from_txn_value_for_cow!(str, String, into_str); +impl_from_txn_value_for_cow!([u8], Vec, into_bytes); + /// Typed adapter over a [`ResultIter`] that yields projected values instead of raw tuples. /// /// This adapts a raw ORM result iterator into scalar projected values. @@ -2974,7 +3030,7 @@ pub trait DecimalType {} #[doc(hidden)] pub fn take_value_at( - tuple: &mut TupleImpl, + tuple: &mut ResultRow<'_, '_>, index: Option, field_name: &str, ) -> Result { @@ -2982,11 +3038,10 @@ pub fn take_value_at( name: field_name.to_string(), span: None, })?; - let value = tuple.get_mut(idx).ok_or(DatabaseError::MisMatch( + let value = tuple.take_value(idx).ok_or(DatabaseError::MisMatch( "the query result schema", "the query result tuple", ))?; - let value = std::mem::replace(value, DataValue::Null); let value = match T::logical_type() { Some(ty) => value.cast(&ty)?, None => value, @@ -2995,6 +3050,28 @@ pub fn take_value_at( T::from_data_value(value) } +#[doc(hidden)] +pub fn take_txn_value_at<'txn, T: FromTxnValue<'txn>>( + row: &mut ResultRow<'_, 'txn>, + index: Option, + field_name: &str, +) -> Result { + let idx = index.ok_or_else(|| DatabaseError::ColumnNotFound { + name: field_name.to_string(), + span: None, + })?; + let value = row.take_txn_value(idx).ok_or(DatabaseError::MisMatch( + "the query result schema", + "the query result tuple", + ))?; + let value = match T::logical_type() { + Some(ty) => value.cast(&ty)?, + None => value, + }; + + T::from_txn_value(value) +} + macro_rules! impl_from_data_value_by_method { ($ty:ty, $method:ident) => { impl FromDataValue for $ty { @@ -3067,7 +3144,7 @@ macro_rules! impl_model_column_type { }; } -impl_model_column_type!(LogicalType::Varbinary(None); Vec); +impl_model_column_type!(LogicalType::Varbinary(None); Vec, Cow<'_, [u8]>); impl_model_column_type!(LogicalType::Boolean; bool); impl_model_column_type!(LogicalType::Tinyint; i8); impl_model_column_type!(LogicalType::Smallint; i16); @@ -3081,10 +3158,16 @@ impl_model_column_type!(LogicalType::Float; f32); impl_model_column_type!(LogicalType::Double; f64); #[cfg(feature = "decimal")] impl_model_column_type!(LogicalType::Decimal(None, None); Decimal); -impl_model_column_type!(LogicalType::Varchar(None, CharLengthUnits::Characters); String, Arc); +impl_model_column_type!( + LogicalType::Varchar(None, CharLengthUnits::Characters); + String, + Arc, + Cow<'_, str> +); impl StringType for String {} impl StringType for Arc {} +impl StringType for Cow<'_, str> {} #[cfg(feature = "decimal")] impl DecimalType for Decimal {} @@ -3129,7 +3212,7 @@ impl FromDataValue for Vec { fn from_data_value(value: DataValue) -> Result { if let DataValue::Binary { value, .. } = value { - Ok(value) + Ok(value.into_vec()) } else { Err(invalid_from_data_value::(&value)) } @@ -3143,7 +3226,7 @@ impl FromDataValue for String { fn from_data_value(value: DataValue) -> Result { if let DataValue::Utf8 { value, .. } = value { - Ok(value) + Ok(value.into_string()) } else { Err(invalid_from_data_value::(&value)) } @@ -3157,7 +3240,7 @@ impl FromDataValue for Arc { fn from_data_value(value: DataValue) -> Result { if let DataValue::Utf8 { value, .. } = value { - Ok(value.into()) + Ok(value.as_str().into()) } else { Err(invalid_from_data_value::(&value)) } @@ -3182,6 +3265,18 @@ impl ToDataValue for &str { } } +impl ToDataValue for Cow<'_, str> { + fn to_data_value(&self) -> DataValue { + DataValue::from(self.to_string()) + } +} + +impl ToDataValue for Cow<'_, [u8]> { + fn to_data_value(&self) -> DataValue { + DataValue::from(self.to_vec()) + } +} + impl FromDataValue for Option { fn logical_type() -> Option { T::logical_type() @@ -3226,7 +3321,7 @@ macro_rules! impl_from_query_tuple { $($name: FromDataValue,)+ { #[allow(non_snake_case)] - fn from_query_tuple(tuple: &mut TupleImpl) -> Result { + fn from_query_tuple(tuple: &mut ResultRow<'_, '_>) -> Result { let expected_len = [$(stringify!($name)),+].len(); if tuple.len() != expected_len { return Err(DatabaseError::MisMatch( @@ -3340,10 +3435,8 @@ fn invalid_from_data_value(value: &DataValue) -> DatabaseError { )) } -fn take_projected_value(tuple: &mut TupleImpl, index: usize) -> Option { - tuple - .get_mut(index) - .map(|value| std::mem::replace(value, DataValue::Null)) +fn take_projected_value(tuple: &mut ResultRow<'_, '_>, index: usize) -> Option { + tuple.take_value(index) } fn extract_projected_data_value( @@ -3357,7 +3450,9 @@ fn extract_projected_data_value( convert_projected_value::(value) } -fn extract_value_from_tuple(tuple: &mut TupleImpl) -> Result { +fn extract_value_from_tuple( + tuple: &mut ResultRow<'_, '_>, +) -> Result { let value = if tuple.len() == 1 { take_projected_value(tuple, 0).expect("checked one projected expression") } else { @@ -3370,7 +3465,9 @@ fn extract_value_from_tuple(tuple: &mut TupleImpl) -> Result(value) } -fn extract_projected_tuple(tuple: &mut TupleImpl) -> Result { +fn extract_projected_tuple( + tuple: &mut ResultRow<'_, '_>, +) -> Result { T::from_query_tuple(tuple) } @@ -3406,9 +3503,17 @@ fn orm_get<'a, E: BindSource<'a>, M: Model>( executor: E, key: &M::PrimaryKey, ) -> Result, DatabaseError> { + extract_optional_model(bind_get::<_, M>(executor, key)?) +} + +/// Binds the lookup of one `M` by primary key. +fn bind_get<'a, E: BindSource<'a>, M: Model>( + executor: E, + key: &M::PrimaryKey, +) -> Result { let primary_key = M::primary_key_field(); let key = key.to_data_value(); - extract_optional_model(bind_orm_context(executor, true, |ctx| { + bind_orm_context(executor, true, |ctx| { let plan: LogicalPlan = ctx .from::()? .filter(|expr| { @@ -3420,17 +3525,7 @@ fn orm_get<'a, E: BindSource<'a>, M: Model>( })? .finish()?; Ok(plan) - })?) -} - -fn orm_list<'a, E: BindSource<'a>, M: Model>( - executor: E, -) -> Result, DatabaseError> { - Ok(bind_orm_context(executor, true, |ctx| { - let plan: LogicalPlan = ctx.from::()?.finish()?; - Ok(plan) - })? - .orm::()) + }) } // GRCOV_EXCL_START @@ -3438,7 +3533,7 @@ fn orm_list<'a, E: BindSource<'a>, M: Model>( mod tests { use super::*; use crate::expression::BinaryOperator; - use crate::types::tuple::{Schema, SchemaView, Tuple}; + use crate::types::tuple::{Schema, SchemaView, Tuple, TupleImpl}; use std::sync::Arc; #[derive(Debug, PartialEq, Eq)] @@ -3498,7 +3593,7 @@ mod tests { impl FromQueryRow for OrmUnitUser { fn from_query_row( schema: &SchemaView<'_, '_>, - tuple: &mut TupleImpl, + tuple: &mut ResultRow<'_, '_>, ) -> Result { Ok(Self { id: take_value_at(tuple, schema.position("id"), "id")?, @@ -3508,6 +3603,15 @@ mod tests { } } + impl<'txn> FromTxnRow<'txn> for OrmUnitUser { + fn from_txn_row( + schema: &SchemaView<'_, '_>, + row: &mut ResultRow<'_, 'txn>, + ) -> Result { + Self::from_query_row(schema, row) + } + } + impl Model for OrmUnitUser { type PrimaryKey = i32; @@ -3586,7 +3690,7 @@ mod tests { impl FromQueryRow for OrmUnitOrder { fn from_query_row( schema: &SchemaView<'_, '_>, - tuple: &mut TupleImpl, + tuple: &mut ResultRow<'_, '_>, ) -> Result { Ok(Self { id: take_value_at(tuple, schema.position("id"), "id")?, @@ -3714,7 +3818,7 @@ mod tests { ) .into(); - let column = DescribeColumn::from_query_row(&schema_view, &mut tuple)?; + let column = DescribeColumn::from_query_row(&schema_view, &mut ResultRow::new(&mut tuple))?; assert_eq!( column, DescribeColumn { @@ -3796,10 +3900,11 @@ mod tests { vec![DataValue::Int8(7), DataValue::from("seven".to_string())], ) .into(); - assert_eq!(take_value_at::(&mut tuple, Some(0), "id")?, 7); - assert!(matches!(tuple[0], DataValue::Null)); + let mut row = ResultRow::new(&mut tuple); + assert_eq!(take_value_at::(&mut row, Some(0), "id")?, 7); + assert!(matches!(row[0], DataValue::Null)); assert!(matches!( - take_value_at::(&mut tuple, None, "missing"), + take_value_at::(&mut row, None, "missing"), Err(DatabaseError::ColumnNotFound { .. }) )); @@ -3808,14 +3913,15 @@ mod tests { vec![DataValue::Int32(1), DataValue::from("one".to_string())], ) .into(); - let projected = <(i32, String) as FromQueryTuple>::from_query_tuple(&mut tuple)?; + let projected = + <(i32, String) as FromQueryTuple>::from_query_tuple(&mut ResultRow::new(&mut tuple))?; assert_eq!(projected, (1, "one".to_string())); assert!(tuple.iter().all(|value| matches!(value, DataValue::Null))); let mut too_wide: TupleImpl = Tuple::new(None, vec![DataValue::Int32(1), DataValue::Int32(2)]).into(); assert!(matches!( - extract_value_from_tuple::(&mut too_wide), + extract_value_from_tuple::(&mut ResultRow::new(&mut too_wide)), Err(DatabaseError::MisMatch( "one projected expression", "the query result" @@ -3824,7 +3930,7 @@ mod tests { let mut too_narrow: TupleImpl = Tuple::new(None, vec![DataValue::Int32(1)]).into(); assert!(matches!( - <(i32, i32) as FromQueryTuple>::from_query_tuple(&mut too_narrow), + <(i32, i32) as FromQueryTuple>::from_query_tuple(&mut ResultRow::new(&mut too_narrow)), Err(DatabaseError::MisMatch( "the expected tuple projection width", "the query result" diff --git a/src/python.rs b/src/python.rs index 3c54a28c..215e3ea4 100644 --- a/src/python.rs +++ b/src/python.rs @@ -14,7 +14,7 @@ #![cfg(all(not(target_arch = "wasm32"), feature = "python"))] -use crate::db::{DataBaseBuilder, Database, DatabaseIter}; +use crate::db::{DataBaseBuilder, Database, DatabaseIter, ResultRow}; use crate::errors::DatabaseError; #[cfg(feature = "lmdb")] use crate::storage::lmdb::LmdbStorage; @@ -169,7 +169,7 @@ enum PythonResultIterInner { impl PythonResultIterInner { fn next_tuple( &mut self, - f: impl FnOnce(&mut TupleImpl) -> R, + f: impl FnOnce(&mut ResultRow<'_, '_>) -> R, ) -> Result, DatabaseError> { match self { #[cfg(feature = "lmdb")] diff --git a/src/serdes/data_value.rs b/src/serdes/data_value.rs index 16a51b17..51dda37d 100644 --- a/src/serdes/data_value.rs +++ b/src/serdes/data_value.rs @@ -212,10 +212,13 @@ impl DataValue { } else { None }; - Ok(DataValue::Binary { value, len }) + Ok(DataValue::Binary { + value: value.into(), + len, + }) } TAG_UTF8 => Ok(DataValue::Utf8 { - value: read_string(reader)?, + value: read_string(reader)?.into(), ty: read_utf8_type(reader)?, unit: read_char_length_units(reader)?, }), @@ -438,7 +441,7 @@ pub(crate) mod test { fn test_serialization() -> Result<(), DatabaseError> { let sources = vec![ DataValue::Binary { - value: vec![0, 255, 128], + value: vec![0, 255, 128].into(), len: Some(8), }, DataValue::from(Vec::::new()), @@ -455,12 +458,12 @@ pub(crate) mod test { DataValue::UInt32(32), DataValue::UInt64(64), DataValue::Utf8 { - value: "hello".to_string(), + value: "hello".into(), ty: Utf8Type::Variable(Some(16)), unit: CharLengthUnits::Characters, }, DataValue::Utf8 { - value: "octets".to_string(), + value: "octets".into(), ty: Utf8Type::Fixed(8), unit: CharLengthUnits::Octets, }, diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs index 43920684..3aa5dfb8 100644 --- a/src/storage/lmdb.rs +++ b/src/storage/lmdb.rs @@ -18,6 +18,7 @@ use crate::storage::{ bounds_contain, bytes_bound_as_slice, owned_bound, InnerIter, KeyValueRef, Storage, StorageTransaction, TransactionIsolationLevel, }; +use crate::types::mapped::MappedScope; use lmdb::{ Cursor, Database, DatabaseFlags, Environment, EnvironmentFlags, RoCursor, RoTransaction, RwTransaction, Transaction as _, WriteFlags, @@ -133,12 +134,16 @@ impl Storage for LmdbStorage { read_only: bool, ) -> Result, DatabaseError> { self.validate_transaction_isolation(isolation)?; - let tx = if read_only { - LmdbTxn::Ro(self.env.begin_ro_txn()?) + let (tx, mapped) = if read_only { + // SAFETY: a read-only transaction reads its snapshot of the memory map, which stays + // mapped and unchanged until the transaction ends, and the scope is dropped first. + let mapped = unsafe { MappedScope::new() }; + (LmdbTxn::Ro(self.env.begin_ro_txn()?), Some(mapped)) } else { - LmdbTxn::Rw(self.env.begin_rw_txn()?) + (LmdbTxn::Rw(self.env.begin_rw_txn()?), None) }; Ok(LmdbTransaction { + mapped, tx, db: self.db, statements: 0, @@ -168,6 +173,8 @@ impl Storage for LmdbStorage { } pub struct LmdbTransaction<'env> { + // Declared before `tx` so it is dropped before the snapshot its reads point into. + mapped: Option, tx: LmdbTxn<'env>, db: Database, statements: u64, @@ -364,6 +371,10 @@ impl StorageTransaction for LmdbTransaction<'_> { } } + fn mapped_scope(&self) -> Option<&MappedScope> { + self.mapped.as_ref() + } + fn point_cursor(&self) -> Result, DatabaseError> { Ok(self.tx.open_ro_cursor(self.db)?) } @@ -426,7 +437,9 @@ impl StorageTransaction for LmdbTransaction<'_> { } fn commit(self) -> Result<(), DatabaseError> { - match self.tx { + let LmdbTransaction { mapped, tx, .. } = self; + drop(mapped); + match tx { LmdbTxn::Ro(tx) => tx.commit()?, LmdbTxn::Rw(tx) => tx.commit()?, } @@ -438,10 +451,140 @@ impl StorageTransaction for LmdbTransaction<'_> { #[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::{LmdbConfig, LmdbStorage}; + use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::db::{CatalogKind, DataBaseBuilder}; + use crate::planner::{PlanArena, TableArenaCell}; + use crate::storage::table_codec::TableCodec; + use crate::storage::{Iter, ScanColumn, Storage, StorageTransaction, TupleDeserializers}; + use crate::types::tuple::Tuple; + use crate::types::value::{DataValue, Utf8Type}; + use crate::types::{CharLengthUnits, LogicalType}; use lmdb::EnvironmentFlags; + use std::borrow::Cow; use tempfile::TempDir; + #[test] + fn read_transactions_map_decoded_strings() { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let storage = LmdbStorage::new(temp_dir.path().join("kite_sql.lmdb")).unwrap(); + let isolation = storage.default_transaction_isolation(); + let types = [ + LogicalType::Varchar(None, CharLengthUnits::Characters), + LogicalType::Char(4, CharLengthUnits::Characters), + ]; + let table_arena = TableArenaCell::default(); + let mut arena = PlanArena::new(&table_arena); + let columns = types + .iter() + .map(|ty| ScanColumn { + column: arena.alloc_column(ColumnCatalog::new( + "c".to_string(), + false, + ColumnDesc::new(ty.clone(), None, false, None).unwrap(), + )), + deserializer: ty.serializable(), + pinned: true, + }) + .collect(); + let deserializers = TupleDeserializers { + columns, + total_len: types.len(), + pk_ty: None, + }; + let mut codec = TableCodec::default(); + let mut writer = storage.transaction().unwrap(); + let row = Tuple::new( + Some(DataValue::Int32(1)), + vec![ + DataValue::from("kite".to_string()), + DataValue::Utf8 { + value: "db".into(), + ty: Utf8Type::Fixed(4), + unit: CharLengthUnits::Characters, + }, + ], + ); + let serializers = types.iter().map(LogicalType::serializable); + writer + .append_tuple(&mut codec, 0, &row.into(), serializers, false) + .unwrap(); + writer.commit().unwrap(); + + // Only a read-only transaction keeps its reads mapped until it ends. + for read_only in [false, true] { + let tx = storage + .transaction_with_isolation(isolation, read_only) + .unwrap(); + let mut iter = tx + .read(&mut codec, &deserializers, 0, (None, None)) + .unwrap(); + let mut tuple = Tuple::default(); + assert!(iter.next_tuple_into(&mut codec, &mut tuple).unwrap()); + for (value, expected) in tuple.values.iter().zip(["kite", "db"]) { + let DataValue::Utf8 { value, .. } = value else { + panic!("expected a string, got {value:?}") + }; + assert_eq!(value, expected); + assert_eq!(value.is_mapped(), read_only); + } + } + } + + #[test] + fn transaction_rows_lend_strings_until_the_transaction_ends() { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut db = DataBaseBuilder::path(temp_dir.path()).build_lmdb().unwrap(); + db.ddl("create table t (a int primary key, b varchar)") + .unwrap(); + db.run("insert into t values (1, 'kite')") + .unwrap() + .done() + .unwrap(); + + // Only a read-only transaction's reads stay mapped; a computed string is always owned. + for read_only in [false, true] { + let mut tx = db.new_transaction(read_only).unwrap(); + let mut iter = tx.run("select b, upper(b) from t").unwrap(); + let (b, upper) = iter + .next_tuple(|_, row| { + let b = row.take_txn_value(0).unwrap().into_str().unwrap(); + let upper = row.take_txn_value(1).unwrap().into_str().unwrap(); + (b, upper) + }) + .unwrap() + .unwrap(); + iter.done().unwrap(); + assert_eq!((b.as_ref(), upper.as_ref()), ("kite", "KITE")); + assert_eq!(matches!(b, Cow::Borrowed(_)), read_only); + assert!(matches!(upper, Cow::Owned(_))); + drop((b, upper)); + tx.commit().unwrap(); + } + } + + #[test] + fn rows_from_read_transactions_outlive_them() { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut db = DataBaseBuilder::path(temp_dir.path()).build_lmdb().unwrap(); + db.ddl("create table t (a int primary key, b varchar)") + .unwrap(); + db.run("insert into t values (1, 'kite')") + .unwrap() + .done() + .unwrap(); + + let mut reader = db.new_transaction(true).unwrap(); + let mut iter = reader.run("select b from t").unwrap(); + let row = iter.next_tuple(|_, row| row.take()).unwrap().unwrap(); + iter.done().unwrap(); + reader.commit().unwrap(); + let DataValue::Utf8 { value, .. } = &row[0] else { + panic!("expected a string") + }; + assert!(!value.is_mapped()); + assert_eq!(value, "kite"); + } + #[test] fn lmdb_backend_smoke() { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); diff --git a/src/storage/mod.rs b/src/storage/mod.rs index bd43bb7c..fdaae9aa 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -35,6 +35,7 @@ use crate::planner::operator::alter_table::change_column::{DefaultChange, NotNul use crate::planner::{MetaArena, TableArenaCell}; use crate::storage::table_codec::{Bytes, IndexStatisticsCodecType, TableCodec, BOUND_MAX_TAG}; use crate::types::index::{Index, IndexId, IndexMeta, IndexMetaRef, IndexType, RuntimeIndexProbe}; +use crate::types::mapped::MappedScope; use crate::types::serialize::TupleValueSerializableImpl; use crate::types::tuple::{Tuple, TupleId, TupleImpl, TupleLike}; use crate::types::value::{DataValue, OrderedMapping, TupleMappingRef}; @@ -225,6 +226,12 @@ pub trait StorageTransaction: Sized { Ok(0) } + /// The scope of a transaction whose reads stay valid, unchanged and at the same address + /// until it ends. Strings decoded from its reads then point into them instead of copying. + fn mapped_scope(&self) -> Option<&MappedScope> { + None + } + /// The bounds is applied to the whole data batches, not per batch. fn read<'a>( &'a self, @@ -243,6 +250,7 @@ pub trait StorageTransaction: Sized { bounds: IterBounds::new(offset, bounds.1), deserializers, iter, + mapped: self.mapped_scope(), }) }) } @@ -1460,14 +1468,18 @@ impl IndexImplParams<'_, T> { let Some(bytes) = self.tx.get_borrowed(key)? else { return Ok(false); }; - TableCodec::decode_tuple_into( - tuple, - self.deserializers.deserializers(), - Some(tuple_id.clone()), - bytes.as_ref(), - self.deserializers.total_len, - pool, - )?; + // SAFETY: `bytes` was read from `self.tx`. + unsafe { + TableCodec::decode_tuple_into( + tuple, + self.deserializers.deserializers(), + Some(tuple_id.clone()), + bytes.as_ref(), + self.deserializers.total_len, + pool, + self.tx.mapped_scope(), + )? + }; Ok(true) }) } @@ -1605,14 +1617,18 @@ impl IndexImpl for PrimaryKeyIndexImpl { .with_pk .then(|| TableCodec::decode_tuple_key(key, ¶ms.index_meta().pk_ty)) .transpose()?; - TableCodec::decode_tuple_into( - tuple, - params.deserializers.deserializers(), - tuple_id, - value, - params.deserializers.total_len, - table_codec.bytes_pool(), - ) + // SAFETY: index iterators only scan ranges of `params.tx`, so `value` was read from it. + unsafe { + TableCodec::decode_tuple_into( + tuple, + params.deserializers.deserializers(), + tuple_id, + value, + params.deserializers.total_len, + table_codec.bytes_pool(), + params.tx.mapped_scope(), + ) + } } fn eq_to_res<'a>( @@ -1633,14 +1649,18 @@ impl IndexImpl for PrimaryKeyIndexImpl { let Some(bytes) = bytes else { return Ok(false); }; - TableCodec::decode_tuple_into( - tuple, - params.deserializers.deserializers(), - params.with_pk.then(|| value.clone()), - bytes.as_ref(), - params.deserializers.total_len, - pool, - )?; + // SAFETY: `bytes` was read from `params.tx`. + unsafe { + TableCodec::decode_tuple_into( + tuple, + params.deserializers.deserializers(), + params.with_pk.then(|| value.clone()), + bytes.as_ref(), + params.deserializers.total_len, + pool, + params.tx.mapped_scope(), + )? + }; Ok(true) })?; Ok(if found { @@ -2016,6 +2036,8 @@ pub struct TupleIter<'a, T: StorageTransaction + 'a> { bounds: IterBounds, deserializers: &'a TupleDeserializers, iter: T::IterType<'a>, + // The scope of the transaction `iter` reads. + mapped: Option<&'a MappedScope>, } impl<'a, T: StorageTransaction + 'a> TupleIter<'a, T> { @@ -2028,6 +2050,7 @@ impl<'a, T: StorageTransaction + 'a> TupleIter<'a, T> { self.iter = table_codec.with_tuple_bound(self.table_id, |min, max, _| { tx.range(Bound::Included(min), Bound::Included(max)) })?; + self.mapped = tx.mapped_scope(); self.bounds = IterBounds::new(bounds.0.unwrap_or(0), bounds.1); Ok(()) } @@ -2052,14 +2075,18 @@ impl<'a, T: StorageTransaction + 'a> Iter for TupleIter<'a, T> { } else { None }; - TableCodec::decode_tuple_into( - tuple, - self.deserializers.deserializers(), - tuple_id, - value, - self.deserializers.total_len, - table_codec.bytes_pool(), - )?; + // SAFETY: `value` was read from the transaction `self.mapped` belongs to. + unsafe { + TableCodec::decode_tuple_into( + tuple, + self.deserializers.deserializers(), + tuple_id, + value, + self.deserializers.total_len, + table_codec.bytes_pool(), + self.mapped, + )? + }; return Ok(true); } diff --git a/src/storage/table_codec.rs b/src/storage/table_codec.rs index 85ea586b..d4a46d32 100644 --- a/src/storage/table_codec.rs +++ b/src/storage/table_codec.rs @@ -25,6 +25,7 @@ use crate::planner::MetaArena; use crate::serdes::{ReferenceDecodeContext, ReferenceSerialization}; use crate::storage::StorageTransaction; use crate::types::index::{Index, IndexId, IndexMeta, IndexType, INDEX_ID_LEN}; +use crate::types::mapped::MappedScope; use crate::types::serialize::{BytesPool, TupleValueSerializableImpl}; use crate::types::tuple::{Tuple, TupleId, TupleLike}; use crate::types::value::{DataValue, IndexKeyMapping}; @@ -828,21 +829,30 @@ impl TableCodec { DataValue::memcomparable_decode(&mut Cursor::new(&bytes[TUPLE_KEY_PREFIX_LEN..]), pk_ty) } + /// Decodes a stored row. With a `scope` strings and binaries point into `bytes`, otherwise + /// they are copied. + /// + /// # Safety + /// + /// With a `scope`, `bytes` must be a read of the transaction holding it. #[inline] - pub(crate) fn decode_tuple_into( + pub(crate) unsafe fn decode_tuple_into( tuple: &mut Tuple, deserializers: I, tuple_id: Option, bytes: &[u8], total_len: usize, pool: &mut BytesPool, + scope: Option<&MappedScope>, ) -> Result<(), DatabaseError> where I: IntoIterator, S: Borrow, { tuple.pk = tuple_id; - tuple.deserialize_from_into(deserializers, Self::strip_stamp(bytes)?, total_len, pool) + let bytes = Self::strip_stamp(bytes)?; + // SAFETY: the caller guarantees `bytes` is a read of `scope`'s transaction. + unsafe { tuple.deserialize_from_into(deserializers, bytes, total_len, pool, scope) } } fn encode_index_meta_value_into( @@ -1225,14 +1235,18 @@ mod tests { .collect_vec(); let mut tuple = Tuple::default(); - TableCodec::decode_tuple_into( - &mut tuple, - &deserializers, - None, - &bytes, - 2, - &mut crate::types::serialize::BytesPool::default(), - )?; + // SAFETY: without a scope every value is copied. + unsafe { + TableCodec::decode_tuple_into( + &mut tuple, + &deserializers, + None, + &bytes, + 2, + &mut crate::types::serialize::BytesPool::default(), + None, + )? + }; assert_eq!( tuple, Tuple::new( diff --git a/src/types/evaluator/binary.rs b/src/types/evaluator/binary.rs index ace5a142..4216169f 100644 --- a/src/types/evaluator/binary.rs +++ b/src/types/evaluator/binary.rs @@ -1010,7 +1010,7 @@ mod test { #[test] fn test_binary_eval_rejects_invalid_positions_and_params() { let value = DataValue::Utf8 { - value: "abc".to_string(), + value: "abc".into(), ty: crate::types::value::Utf8Type::Variable(None), unit: crate::types::CharLengthUnits::Characters, }; diff --git a/src/types/evaluator/boolean.rs b/src/types/evaluator/boolean.rs index 3d4e8fd9..bf1a1e73 100644 --- a/src/types/evaluator/boolean.rs +++ b/src/types/evaluator/boolean.rs @@ -230,7 +230,7 @@ mod test { assert_eq!( boolean_to_char_cast_eval(4, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "true".to_string(), + value: "true".into(), ty: Utf8Type::Fixed(4), unit: CharLengthUnits::Characters, } @@ -239,7 +239,7 @@ mod test { boolean_to_varchar_cast_eval(Some(4), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "true".to_string(), + value: "true".into(), ty: Utf8Type::Variable(Some(4)), unit: CharLengthUnits::Characters, } @@ -256,7 +256,7 @@ mod test { ) .unwrap(), DataValue::Utf8 { - value: "false".to_string(), + value: "false".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/cast.rs b/src/types/evaluator/cast.rs index 11eb28cc..56be76c0 100644 --- a/src/types/evaluator/cast.rs +++ b/src/types/evaluator/cast.rs @@ -39,7 +39,7 @@ use crate::types::evaluator::uint64::*; use crate::types::evaluator::uint8::*; use crate::types::evaluator::utf8::*; use crate::types::evaluator::{CastEvaluatorParams, CastEvaluatorRef}; -use crate::types::value::{DataValue, Utf8Type}; +use crate::types::value::{BinaryBytes, DataValue, Utf8Str, Utf8Type}; use crate::types::CharLengthUnits; use crate::types::LogicalType; use paste::paste; @@ -128,10 +128,11 @@ fn is_chrono_type_code(code: u16) -> bool { } pub(crate) fn to_char( - value: String, + value: impl Into, len: u32, unit: CharLengthUnits, ) -> Result { + let value = value.into(); if DataValue::check_string_len(&value, len as usize, unit) { return Err(DatabaseError::TooLong); } @@ -143,17 +144,24 @@ pub(crate) fn to_char( }) } -pub(crate) fn to_varbinary(value: Vec, len: Option) -> Result { - let result = DataValue::Binary { value, len }; +pub(crate) fn to_varbinary( + value: impl Into, + len: Option, +) -> Result { + let result = DataValue::Binary { + value: value.into(), + len, + }; result.check_len(&LogicalType::Varbinary(len))?; Ok(result) } pub(crate) fn to_varchar( - value: String, + value: impl Into, len: Option, unit: CharLengthUnits, ) -> Result { + let value = value.into(); if let Some(len) = len { if DataValue::check_string_len(&value, len as usize, unit) { return Err(DatabaseError::TooLong); @@ -1316,7 +1324,7 @@ mod test { fn utf8(value: &str) -> DataValue { DataValue::Utf8 { - value: value.to_string(), + value: value.to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } @@ -1444,7 +1452,7 @@ mod test { ( LogicalType::Char(4, CharLengthUnits::Characters), DataValue::Utf8 { - value: "true".to_string(), + value: "true".into(), ty: Utf8Type::Fixed(4), unit: CharLengthUnits::Characters, }, @@ -1452,7 +1460,7 @@ mod test { ( LogicalType::Varchar(Some(4), CharLengthUnits::Characters), DataValue::Utf8 { - value: "true".to_string(), + value: "true".into(), ty: Utf8Type::Variable(Some(4)), unit: CharLengthUnits::Characters, }, @@ -1501,7 +1509,7 @@ mod test { ( LogicalType::Char(3, CharLengthUnits::Characters), DataValue::Utf8 { - value: "1.5".to_string(), + value: "1.5".into(), ty: Utf8Type::Fixed(3), unit: CharLengthUnits::Characters, }, @@ -1509,7 +1517,7 @@ mod test { ( LogicalType::Varchar(Some(3), CharLengthUnits::Characters), DataValue::Utf8 { - value: "1.5".to_string(), + value: "1.5".into(), ty: Utf8Type::Variable(Some(3)), unit: CharLengthUnits::Characters, }, @@ -1538,7 +1546,7 @@ mod test { ( LogicalType::Char(3, CharLengthUnits::Characters), DataValue::Utf8 { - value: "1.5".to_string(), + value: "1.5".into(), ty: Utf8Type::Fixed(3), unit: CharLengthUnits::Characters, }, @@ -1546,7 +1554,7 @@ mod test { ( LogicalType::Varchar(Some(3), CharLengthUnits::Characters), DataValue::Utf8 { - value: "1.5".to_string(), + value: "1.5".into(), ty: Utf8Type::Variable(Some(3)), unit: CharLengthUnits::Characters, }, @@ -1666,7 +1674,7 @@ mod test { DataValue::Int32(9), )?, DataValue::Utf8 { - value: "9".to_string(), + value: "9".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, } @@ -1678,7 +1686,7 @@ mod test { DataValue::UInt32(10), )?, DataValue::Utf8 { - value: "10".to_string(), + value: "10".into(), ty: Utf8Type::Variable(Some(2)), unit: CharLengthUnits::Characters, } @@ -1727,7 +1735,7 @@ mod test { utf8("中\0") )?, DataValue::Binary { - value: "中\0".as_bytes().to_vec(), + value: "中\0".as_bytes().to_vec().into(), len: Some(4) } ); @@ -1828,13 +1836,13 @@ mod test { LogicalType::Char(2, CharLengthUnits::Characters), LogicalType::Varchar(Some(2), CharLengthUnits::Characters), DataValue::Utf8 { - value: "ab".to_string(), + value: "ab".into(), ty: Utf8Type::Fixed(2), unit: CharLengthUnits::Characters, }, )?, DataValue::Utf8 { - value: "ab".to_string(), + value: "ab".into(), ty: Utf8Type::Variable(Some(2)), unit: CharLengthUnits::Characters, } @@ -1846,7 +1854,7 @@ mod test { utf8("ab"), )?, DataValue::Utf8 { - value: "ab".to_string(), + value: "ab".into(), ty: Utf8Type::Fixed(2), unit: CharLengthUnits::Characters, } @@ -1931,7 +1939,7 @@ mod test { date.clone(), )?, DataValue::Utf8 { - value: "2024-01-02".to_string(), + value: "2024-01-02".into(), ty: Utf8Type::Fixed(10), unit: CharLengthUnits::Characters, } @@ -1943,7 +1951,7 @@ mod test { date.clone(), )?, DataValue::Utf8 { - value: "2024-01-02".to_string(), + value: "2024-01-02".into(), ty: Utf8Type::Variable(Some(10)), unit: CharLengthUnits::Characters, } @@ -1972,7 +1980,7 @@ mod test { datetime.clone(), )?, DataValue::Utf8 { - value: "2024-01-02 03:04:05".to_string(), + value: "2024-01-02 03:04:05".into(), ty: Utf8Type::Fixed(19), unit: CharLengthUnits::Characters, } @@ -1984,7 +1992,7 @@ mod test { datetime.clone(), )?, DataValue::Utf8 { - value: "2024-01-02 03:04:05".to_string(), + value: "2024-01-02 03:04:05".into(), ty: Utf8Type::Variable(Some(19)), unit: CharLengthUnits::Characters, } @@ -2082,7 +2090,7 @@ mod test { DataValue::Time32(DataValue::pack_time(3 * 3600 + 4 * 60 + 5, 0, 0), 0), )?, DataValue::Utf8 { - value: "03:04:05".to_string(), + value: "03:04:05".into(), ty: Utf8Type::Fixed(8), unit: CharLengthUnits::Characters, } @@ -2094,7 +2102,7 @@ mod test { DataValue::Time32(DataValue::pack_time(3 * 3600 + 4 * 60 + 5, 0, 0), 0), )?, DataValue::Utf8 { - value: "03:04:05".to_string(), + value: "03:04:05".into(), ty: Utf8Type::Variable(Some(8)), unit: CharLengthUnits::Characters, } @@ -2106,7 +2114,7 @@ mod test { timestamp.clone(), )?, DataValue::Utf8 { - value: "2024-01-02 03:04:05.123".to_string(), + value: "2024-01-02 03:04:05.123".into(), ty: Utf8Type::Fixed(23), unit: CharLengthUnits::Characters, } @@ -2118,7 +2126,7 @@ mod test { timestamp.clone(), )?, DataValue::Utf8 { - value: "2024-01-02 03:04:05.123".to_string(), + value: "2024-01-02 03:04:05.123".into(), ty: Utf8Type::Variable(Some(23)), unit: CharLengthUnits::Characters, } @@ -2147,7 +2155,7 @@ mod test { assert_eq!( binary.clone().cast(&LogicalType::Varbinary(Some(2)))?, DataValue::Binary { - value: vec![0, 255], + value: vec![0, 255].into(), len: Some(2) } ); @@ -2247,7 +2255,7 @@ mod test { ( LogicalType::Char(4, CharLengthUnits::Characters), DataValue::Utf8 { - value: "12.5".to_string(), + value: "12.5".into(), ty: Utf8Type::Fixed(4), unit: CharLengthUnits::Characters, }, @@ -2255,7 +2263,7 @@ mod test { ( LogicalType::Varchar(Some(4), CharLengthUnits::Characters), DataValue::Utf8 { - value: "12.5".to_string(), + value: "12.5".into(), ty: Utf8Type::Variable(Some(4)), unit: CharLengthUnits::Characters, }, diff --git a/src/types/evaluator/date.rs b/src/types/evaluator/date.rs index 9df5454e..98fe5152 100644 --- a/src/types/evaluator/date.rs +++ b/src/types/evaluator/date.rs @@ -91,7 +91,7 @@ mod test { assert_eq!( date32_to_char_cast_eval(10, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "2024-01-02".to_string(), + value: "2024-01-02".into(), ty: Utf8Type::Fixed(10), unit: CharLengthUnits::Characters, } @@ -100,7 +100,7 @@ mod test { date32_to_varchar_cast_eval(Some(10), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "2024-01-02".to_string(), + value: "2024-01-02".into(), ty: Utf8Type::Variable(Some(10)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/datetime.rs b/src/types/evaluator/datetime.rs index ad8a448d..acfd753d 100644 --- a/src/types/evaluator/datetime.rs +++ b/src/types/evaluator/datetime.rs @@ -117,7 +117,7 @@ mod test { assert_eq!( date64_to_char_cast_eval(19, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "2024-01-02 03:04:05".to_string(), + value: "2024-01-02 03:04:05".into(), ty: Utf8Type::Fixed(19), unit: CharLengthUnits::Characters, } @@ -126,7 +126,7 @@ mod test { date64_to_varchar_cast_eval(Some(19), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "2024-01-02 03:04:05".to_string(), + value: "2024-01-02 03:04:05".into(), ty: Utf8Type::Variable(Some(19)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/decimal.rs b/src/types/evaluator/decimal.rs index c4cf41c1..f1d0c293 100644 --- a/src/types/evaluator/decimal.rs +++ b/src/types/evaluator/decimal.rs @@ -168,7 +168,7 @@ mod test { assert_eq!( decimal_to_char_cast_eval(4, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "12.5".to_string(), + value: "12.5".into(), ty: Utf8Type::Fixed(4), unit: CharLengthUnits::Characters, } @@ -177,7 +177,7 @@ mod test { decimal_to_varchar_cast_eval(Some(4), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "12.5".to_string(), + value: "12.5".into(), ty: Utf8Type::Variable(Some(4)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/float32.rs b/src/types/evaluator/float32.rs index af4e2d69..efb224f9 100644 --- a/src/types/evaluator/float32.rs +++ b/src/types/evaluator/float32.rs @@ -167,7 +167,7 @@ mod test { assert_eq!( float32_to_char_cast_eval(3, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1.5".to_string(), + value: "1.5".into(), ty: Utf8Type::Fixed(3), unit: CharLengthUnits::Characters, } @@ -176,7 +176,7 @@ mod test { float32_to_varchar_cast_eval(Some(3), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "1.5".to_string(), + value: "1.5".into(), ty: Utf8Type::Variable(Some(3)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/float64.rs b/src/types/evaluator/float64.rs index a4ec98c8..260e3d4e 100644 --- a/src/types/evaluator/float64.rs +++ b/src/types/evaluator/float64.rs @@ -335,7 +335,7 @@ mod test { assert_eq!( float64_to_char_cast_eval(3, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1.5".to_string(), + value: "1.5".into(), ty: Utf8Type::Fixed(3), unit: CharLengthUnits::Characters, } @@ -344,7 +344,7 @@ mod test { float64_to_varchar_cast_eval(Some(3), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "1.5".to_string(), + value: "1.5".into(), ty: Utf8Type::Variable(Some(3)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/int16.rs b/src/types/evaluator/int16.rs index 4432bb27..17cc8e8b 100644 --- a/src/types/evaluator/int16.rs +++ b/src/types/evaluator/int16.rs @@ -80,7 +80,7 @@ mod test { assert_eq!( int16_to_char_cast_eval(1, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, } @@ -89,7 +89,7 @@ mod test { int16_to_varchar_cast_eval(Some(1), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Variable(Some(1)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/int32.rs b/src/types/evaluator/int32.rs index 32b97f5b..10f637ba 100644 --- a/src/types/evaluator/int32.rs +++ b/src/types/evaluator/int32.rs @@ -100,7 +100,7 @@ mod test { assert_eq!( int32_to_char_cast_eval(1, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, } @@ -109,7 +109,7 @@ mod test { int32_to_varchar_cast_eval(Some(1), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Variable(Some(1)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/int64.rs b/src/types/evaluator/int64.rs index 2206f6e7..ce26e48e 100644 --- a/src/types/evaluator/int64.rs +++ b/src/types/evaluator/int64.rs @@ -80,7 +80,7 @@ mod test { assert_eq!( int64_to_char_cast_eval(1, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, } @@ -89,7 +89,7 @@ mod test { int64_to_varchar_cast_eval(Some(1), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Variable(Some(1)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/int8.rs b/src/types/evaluator/int8.rs index b7a55756..9aad2190 100644 --- a/src/types/evaluator/int8.rs +++ b/src/types/evaluator/int8.rs @@ -80,7 +80,7 @@ mod test { assert_eq!( int8_to_char_cast_eval(1, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, } @@ -88,7 +88,7 @@ mod test { assert_eq!( int8_to_varchar_cast_eval(Some(1), CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Variable(Some(1)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/time32.rs b/src/types/evaluator/time32.rs index e283e60c..906f9efc 100644 --- a/src/types/evaluator/time32.rs +++ b/src/types/evaluator/time32.rs @@ -281,7 +281,7 @@ mod test { assert_eq!( time32_to_char_cast_eval(12, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "03:04:05.123".to_string(), + value: "03:04:05.123".into(), ty: Utf8Type::Fixed(12), unit: CharLengthUnits::Characters, } @@ -290,7 +290,7 @@ mod test { time32_to_varchar_cast_eval(Some(12), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "03:04:05.123".to_string(), + value: "03:04:05.123".into(), ty: Utf8Type::Variable(Some(12)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/time64.rs b/src/types/evaluator/time64.rs index 9abdb1f9..f0fdcc06 100644 --- a/src/types/evaluator/time64.rs +++ b/src/types/evaluator/time64.rs @@ -238,7 +238,7 @@ mod test { assert_eq!( time64_to_char_cast_eval(23, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "2024-01-02 03:04:05.123".to_string(), + value: "2024-01-02 03:04:05.123".into(), ty: Utf8Type::Fixed(23), unit: CharLengthUnits::Characters, } @@ -247,7 +247,7 @@ mod test { time64_to_varchar_cast_eval(Some(23), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "2024-01-02 03:04:05.123".to_string(), + value: "2024-01-02 03:04:05.123".into(), ty: Utf8Type::Variable(Some(23)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/tuple.rs b/src/types/evaluator/tuple.rs index 5625ba0f..c276b6e9 100644 --- a/src/types/evaluator/tuple.rs +++ b/src/types/evaluator/tuple.rs @@ -210,7 +210,7 @@ mod test { .eval(DataValue::Tuple(vec![ DataValue::Int32(1), DataValue::Utf8 { - value: "2".to_string(), + value: "2".into(), ty: crate::types::value::Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }, diff --git a/src/types/evaluator/uint16.rs b/src/types/evaluator/uint16.rs index fc434942..856e44c5 100644 --- a/src/types/evaluator/uint16.rs +++ b/src/types/evaluator/uint16.rs @@ -79,7 +79,7 @@ mod test { assert_eq!( uint16_to_char_cast_eval(1, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, } @@ -88,7 +88,7 @@ mod test { uint16_to_varchar_cast_eval(Some(1), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Variable(Some(1)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/uint32.rs b/src/types/evaluator/uint32.rs index 64780e8e..f93f5b09 100644 --- a/src/types/evaluator/uint32.rs +++ b/src/types/evaluator/uint32.rs @@ -79,7 +79,7 @@ mod test { assert_eq!( uint32_to_char_cast_eval(1, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, } @@ -88,7 +88,7 @@ mod test { uint32_to_varchar_cast_eval(Some(1), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Variable(Some(1)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/uint64.rs b/src/types/evaluator/uint64.rs index dca97772..3c2442f3 100644 --- a/src/types/evaluator/uint64.rs +++ b/src/types/evaluator/uint64.rs @@ -79,7 +79,7 @@ mod test { assert_eq!( uint64_to_char_cast_eval(1, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, } @@ -88,7 +88,7 @@ mod test { uint64_to_varchar_cast_eval(Some(1), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Variable(Some(1)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/uint8.rs b/src/types/evaluator/uint8.rs index 56da2fa3..9521df67 100644 --- a/src/types/evaluator/uint8.rs +++ b/src/types/evaluator/uint8.rs @@ -79,7 +79,7 @@ mod test { assert_eq!( uint8_to_char_cast_eval(1, CharLengthUnits::Characters, value.clone()).unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, } @@ -88,7 +88,7 @@ mod test { uint8_to_varchar_cast_eval(Some(1), CharLengthUnits::Characters, value.clone()) .unwrap(), DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Variable(Some(1)), unit: CharLengthUnits::Characters, } diff --git a/src/types/evaluator/utf8.rs b/src/types/evaluator/utf8.rs index 8d2fa698..a5890f61 100644 --- a/src/types/evaluator/utf8.rs +++ b/src/types/evaluator/utf8.rs @@ -15,7 +15,7 @@ use crate::errors::DatabaseError; use crate::types::evaluator::cast::{to_char, to_varbinary, to_varchar}; use crate::types::evaluator::DataValue; -use crate::types::value::Utf8Type; +use crate::types::value::{Utf8Str, Utf8Type}; use crate::types::CharLengthUnits; use ordered_float::OrderedFloat; #[cfg(feature = "decimal")] @@ -39,13 +39,13 @@ macro_rules! utf8_binary { }; } -utf8_binary!(utf8_gt_binary_eval, |v1: &String, v2: &String| { +utf8_binary!(utf8_gt_binary_eval, |v1: &Utf8Str, v2: &Utf8Str| { DataValue::Boolean(v1 > v2) }); -utf8_binary!(utf8_gt_eq_binary_eval, |v1: &String, v2: &String| { +utf8_binary!(utf8_gt_eq_binary_eval, |v1: &Utf8Str, v2: &Utf8Str| { DataValue::Boolean(v1 >= v2) }); -utf8_binary!(utf8_lt_binary_eval, |v1: &String, v2: &String| { +utf8_binary!(utf8_lt_binary_eval, |v1: &Utf8Str, v2: &Utf8Str| { DataValue::Boolean(v1 < v2) }); @@ -99,7 +99,7 @@ crate::define_cast_evaluator!( ); crate::define_cast_evaluator!( utf8_to_varbinary_cast_eval { len: Option }, - DataValue::Utf8 { value, .. } => |this| to_varbinary(value.into_bytes(), this.len) + DataValue::Utf8 { value, .. } => |this| to_varbinary(value.into_string().into_bytes(), this.len) ); #[cfg(feature = "time")] @@ -166,7 +166,7 @@ mod chrono_cast { (3 | 6 | 9, false) => crate::types::value::TIME_STAMP_FMT_WITHOUT_ZONE, _ => crate::types::value::TIME_STAMP_FMT_WITH_ZONE, }; - let mut complete_value = value; + let mut complete_value = value.into_string(); if this.zone && !complete_value.contains('+') { complete_value.push_str("+00:00"); } @@ -208,13 +208,13 @@ pub use chrono_cast::*; crate::define_cast_evaluator!(utf8_to_decimal_cast_eval, DataValue::Utf8 { value, .. } => { Ok(DataValue::Decimal(Decimal::from_str(&value)?)) }); -utf8_binary!(utf8_lt_eq_binary_eval, |v1: &String, v2: &String| { +utf8_binary!(utf8_lt_eq_binary_eval, |v1: &Utf8Str, v2: &Utf8Str| { DataValue::Boolean(v1 <= v2) }); -utf8_binary!(utf8_eq_binary_eval, |v1: &String, v2: &String| { +utf8_binary!(utf8_eq_binary_eval, |v1: &Utf8Str, v2: &Utf8Str| { DataValue::Boolean(v1 == v2) }); -utf8_binary!(utf8_not_eq_binary_eval, |v1: &String, v2: &String| { +utf8_binary!(utf8_not_eq_binary_eval, |v1: &Utf8Str, v2: &Utf8Str| { DataValue::Boolean(v1 != v2) }); pub fn utf8_string_concat_binary_eval( @@ -223,7 +223,7 @@ pub fn utf8_string_concat_binary_eval( ) -> Result { Ok(match (left, right) { (DataValue::Utf8 { mut value, .. }, DataValue::Utf8 { value: right, .. }) => { - value.push_str(right); + value.to_mut().push_str(right); DataValue::Utf8 { value, ty: Utf8Type::Variable(None), @@ -363,7 +363,7 @@ mod test { fn utf8(value: &str) -> DataValue { DataValue::Utf8 { - value: value.to_string(), + value: value.to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } @@ -566,7 +566,7 @@ mod test { assert_eq!( utf8_to_char_cast_eval(2, CharLengthUnits::Characters, utf8("ab")).unwrap(), DataValue::Utf8 { - value: "ab".to_string(), + value: "ab".into(), ty: Utf8Type::Fixed(2), unit: CharLengthUnits::Characters, } @@ -574,7 +574,7 @@ mod test { assert_eq!( utf8_to_varchar_cast_eval(Some(2), CharLengthUnits::Characters, utf8("ab")).unwrap(), DataValue::Utf8 { - value: "ab".to_string(), + value: "ab".into(), ty: Utf8Type::Variable(Some(2)), unit: CharLengthUnits::Characters, } diff --git a/src/types/mapped.rs b/src/types/mapped.rs new file mode 100644 index 00000000..b2782dc4 --- /dev/null +++ b/src/types/mapped.rs @@ -0,0 +1,447 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::borrow::{Borrow, Cow}; +use std::cmp::Ordering; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::ops::Deref; +use std::ptr::NonNull; +#[cfg(debug_assertions)] +use std::sync::{Arc, Weak}; +use std::{slice, str}; + +/// Held by a transaction whose reads stay valid, unchanged and at the same address until it +/// ends, such as an LMDB read-only snapshot. Strings and binaries decoded from those reads point +/// into them instead of being copied. +#[derive(Debug)] +pub struct MappedScope { + #[cfg(debug_assertions)] + live: Arc<()>, +} + +impl MappedScope { + pub unsafe fn new() -> Self { + Self { + #[cfg(debug_assertions)] + live: Arc::new(()), + } + } +} + +mod sealed { + pub trait Sealed {} + + impl Sealed for str {} + impl Sealed for [u8] {} +} + +pub trait Mappable: sealed::Sealed + ToOwned { + #[doc(hidden)] + fn as_raw(&self) -> &[u8]; + #[doc(hidden)] + unsafe fn from_raw(bytes: &[u8]) -> &Self; + + #[doc(hidden)] + fn owned_into_raw(owned: Self::Owned) -> Vec; + + #[doc(hidden)] + unsafe fn owned_from_raw(raw: Vec) -> Self::Owned; + + #[doc(hidden)] + fn owned_capacity(owned: &Self::Owned) -> usize; +} + +impl Mappable for str { + fn as_raw(&self) -> &[u8] { + self.as_bytes() + } + + unsafe fn from_raw(bytes: &[u8]) -> &Self { + unsafe { str::from_utf8_unchecked(bytes) } + } + + fn owned_into_raw(owned: String) -> Vec { + owned.into_bytes() + } + + unsafe fn owned_from_raw(raw: Vec) -> String { + unsafe { String::from_utf8_unchecked(raw) } + } + + fn owned_capacity(owned: &String) -> usize { + owned.capacity() + } +} + +impl Mappable for [u8] { + fn as_raw(&self) -> &[u8] { + self + } + + unsafe fn from_raw(bytes: &[u8]) -> &Self { + bytes + } + + fn owned_into_raw(owned: Vec) -> Vec { + owned + } + + unsafe fn owned_from_raw(raw: Vec) -> Vec { + raw + } + + fn owned_capacity(owned: &Vec) -> usize { + owned.capacity() + } +} + +pub type Utf8Str = MaybeMapped; +pub type BinaryBytes = MaybeMapped<[u8]>; + +/// A value owned, or pointing into a [`MappedScope`]'s reads. +/// +/// A mapped value is only valid until its transaction ends, so it never leaves the engine as is: +/// cloning one copies it into an owned value, a result row hands its values out only through +/// copying methods, and an executor must not read a value kept from an earlier execution (see +/// `ExecutorNode::reset`). +pub struct MaybeMapped(Repr); + +enum Repr { + Owned(B::Owned), + Mapped(MappedRead), +} + +#[derive(Clone)] +struct MappedRead { + ptr: NonNull, + len: usize, + #[cfg(debug_assertions)] + live: Weak<()>, +} + +unsafe impl Send for MappedRead {} +unsafe impl Sync for MappedRead {} + +impl MaybeMapped { + #[cfg_attr(not(debug_assertions), expect(unused_variables))] + pub(crate) unsafe fn mapped(value: &B, scope: &MappedScope) -> Self { + let raw = value.as_raw(); + MaybeMapped(Repr::Mapped(MappedRead { + ptr: NonNull::from(raw).cast(), + len: raw.len(), + #[cfg(debug_assertions)] + live: Arc::downgrade(&scope.live), + })) + } + + pub fn into_owned(self) -> B::Owned { + match self.0 { + Repr::Owned(value) => value, + Repr::Mapped(_) => (*self).to_owned(), + } + } + + pub fn to_mut(&mut self) -> &mut B::Owned { + self.materialize(); + let Repr::Owned(value) = &mut self.0 else { + unreachable!("materialize leaves an owned value") + }; + value + } + + #[cfg(feature = "spill")] + pub(crate) fn heap_size(&self) -> usize { + match &self.0 { + Repr::Owned(value) => B::owned_capacity(value), + Repr::Mapped(_) => 0, + } + } + + pub(crate) unsafe fn into_cow<'a>(self) -> Cow<'a, B> { + match self.0 { + Repr::Owned(value) => Cow::Owned(value), + Repr::Mapped(mapped) => { + #[cfg(debug_assertions)] + assert!( + mapped.live.strong_count() > 0, + "mapped value read after its transaction ended" + ); + // SAFETY: the read stays valid during `'a`, as the caller guarantees. + Cow::Borrowed(unsafe { + B::from_raw(slice::from_raw_parts(mapped.ptr.as_ptr(), mapped.len)) + }) + } + } + } + + #[inline] + pub(crate) fn materialize(&mut self) { + if let Repr::Mapped(_) = self.0 { + self.0 = Repr::Owned((**self).to_owned()); + } + } + + pub(crate) unsafe fn clone_with(&self, copy: impl FnOnce(&[u8]) -> Vec) -> Self { + MaybeMapped(match &self.0 { + // SAFETY: the caller guarantees `copy` returns the bytes of a `B`. + Repr::Owned(_) => Repr::Owned(unsafe { B::owned_from_raw(copy((**self).as_raw())) }), + Repr::Mapped(mapped) => Repr::Mapped(mapped.clone()), + }) + } + + pub(crate) fn into_owned_raw(self) -> Option> { + match self.0 { + Repr::Owned(value) => Some(B::owned_into_raw(value)), + Repr::Mapped(_) => None, + } + } + + #[cfg(test)] + pub(crate) fn is_mapped(&self) -> bool { + matches!(self.0, Repr::Mapped(_)) + } +} + +impl Utf8Str { + #[inline] + pub fn as_str(&self) -> &str { + self + } + + pub fn into_string(self) -> String { + self.into_owned() + } +} + +impl BinaryBytes { + #[inline] + pub fn as_bytes(&self) -> &[u8] { + self + } + + pub fn into_vec(self) -> Vec { + self.into_owned() + } +} + +impl Deref for MaybeMapped { + type Target = B; + + #[inline] + fn deref(&self) -> &B { + match &self.0 { + Repr::Owned(value) => value.borrow(), + Repr::Mapped(mapped) => { + #[cfg(debug_assertions)] + assert!( + mapped.live.strong_count() > 0, + "mapped value read after its transaction ended" + ); + // SAFETY: `mapped` was built from a `B` that stays valid while its transaction + // runs, which the caller of `MaybeMapped::mapped` guaranteed. + unsafe { B::from_raw(slice::from_raw_parts(mapped.ptr.as_ptr(), mapped.len)) } + } + } + } +} + +impl Clone for MaybeMapped { + fn clone(&self) -> Self { + MaybeMapped(Repr::Owned((**self).to_owned())) + } +} + +impl Default for MaybeMapped +where + B::Owned: Default, +{ + fn default() -> Self { + MaybeMapped(Repr::Owned(B::Owned::default())) + } +} + +impl AsRef for MaybeMapped { + fn as_ref(&self) -> &B { + self + } +} + +impl Borrow for MaybeMapped { + fn borrow(&self) -> &B { + self + } +} + +impl PartialEq for MaybeMapped { + #[inline] + fn eq(&self, other: &Self) -> bool { + **self == **other + } +} + +impl Eq for MaybeMapped {} + +impl PartialOrd for MaybeMapped { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for MaybeMapped { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + (**self).cmp(&**other) + } +} + +impl Hash for MaybeMapped { + #[inline] + fn hash(&self, state: &mut H) { + (**self).hash(state) + } +} + +impl fmt::Debug for MaybeMapped { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl fmt::Display for Utf8Str { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.as_str(), f) + } +} + +macro_rules! impl_owned_conversions { + ($borrowed:ty, $owned:ty) => { + impl From<$owned> for MaybeMapped<$borrowed> { + fn from(value: $owned) -> Self { + MaybeMapped(Repr::Owned(value)) + } + } + + impl From<&$borrowed> for MaybeMapped<$borrowed> { + fn from(value: &$borrowed) -> Self { + MaybeMapped(Repr::Owned(value.to_owned())) + } + } + + impl From> for $owned { + fn from(value: MaybeMapped<$borrowed>) -> Self { + value.into_owned() + } + } + + impl PartialEq<$borrowed> for MaybeMapped<$borrowed> { + fn eq(&self, other: &$borrowed) -> bool { + **self == *other + } + } + + impl PartialEq<&$borrowed> for MaybeMapped<$borrowed> { + fn eq(&self, other: &&$borrowed) -> bool { + **self == **other + } + } + + impl PartialEq<$owned> for MaybeMapped<$borrowed> { + fn eq(&self, other: &$owned) -> bool { + **self == **other + } + } + }; +} + +impl_owned_conversions!(str, String); +impl_owned_conversions!([u8], Vec); + +// GRCOV_EXCL_START +#[cfg(test)] +mod tests { + use super::{BinaryBytes, Mappable, MappedScope, MaybeMapped, Utf8Str}; + use std::borrow::Borrow; + use std::collections::hash_map::DefaultHasher; + use std::fmt::Debug; + use std::hash::{Hash, Hasher}; + + fn hash(value: &T) -> u64 { + let mut hasher = DefaultHasher::new(); + value.hash(&mut hasher); + hasher.finish() + } + + // `read` stands for a read of the scope's transaction; `next` sorts right after it. + fn check_mapped(read: &B, next: &B) + where + B: ?Sized + Mappable + Ord + Hash + Debug, + B::Owned: Debug, + { + // SAFETY: `read` stays unchanged and outlives the scope. + let scope = unsafe { MappedScope::new() }; + // SAFETY: as above. + let mapped = unsafe { MaybeMapped::mapped(read, &scope) }; + let owned = MaybeMapped::(super::Repr::Owned(read.to_owned())); + + assert!(mapped.is_mapped() && !owned.is_mapped()); + assert_eq!(mapped, owned); + assert_eq!(hash(&mapped), hash(read)); + assert_eq!(format!("{mapped:?}"), format!("{read:?}")); + let next = MaybeMapped::(super::Repr::Owned(next.to_owned())); + assert!(mapped < next); + // A clone may leave the transaction, so it is copied; a shared value keeps pointing into + // the read and has no buffer to recycle. + let clone = mapped.clone(); + assert!(!clone.is_mapped() && clone == owned); + // SAFETY: `to_vec` copies what it is given. + let shared = unsafe { mapped.clone_with(<[u8]>::to_vec) }; + assert!(shared.is_mapped()); + assert_eq!(shared.into_owned_raw(), None); + + // SAFETY: as above. + let mut materialized = unsafe { mapped.clone_with(<[u8]>::to_vec) }; + materialized.materialize(); + assert!(!materialized.is_mapped()); + assert_eq!(materialized, owned); + // SAFETY: as above. + let mut edited = unsafe { mapped.clone_with(<[u8]>::to_vec) }; + let _ = edited.to_mut(); + assert!(!edited.is_mapped()); + assert!(Borrow::::borrow(&mapped.into_owned()) == read); + } + + #[test] + fn mapped_values_behave_like_owned_ones() { + check_mapped("kite", "kitf"); + check_mapped(&b"kite"[..], &b"kitf"[..]); + assert_eq!(Utf8Str::from("kite").to_string(), "kite"); + assert_eq!(BinaryBytes::from(vec![1, 2]).into_vec(), [1, 2]); + } + + #[cfg(debug_assertions)] + #[test] + #[should_panic(expected = "mapped value read after its transaction ended")] + fn reading_after_the_scope_ends_panics() { + let read = String::from("kite"); + // SAFETY: `read` stays unchanged and outlives the scope. + let scope = unsafe { MappedScope::new() }; + // SAFETY: as above; the read below is caught before it dereferences. + let mapped = unsafe { Utf8Str::mapped(read.as_str(), &scope) }; + drop(scope); + let _ = mapped.as_str(); + } +} +// GRCOV_EXCL_STOP diff --git a/src/types/mod.rs b/src/types/mod.rs index a789550f..1ef0fa12 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -15,6 +15,7 @@ pub mod bitset; pub mod evaluator; pub mod index; +pub mod mapped; pub mod serialize; pub mod tuple; pub mod tuple_builder; diff --git a/src/types/serialize.rs b/src/types/serialize.rs index 3d56d34b..20b339a4 100644 --- a/src/types/serialize.rs +++ b/src/types/serialize.rs @@ -13,7 +13,8 @@ // limitations under the License. use crate::errors::DatabaseError; -use crate::types::value::{DataValue, Utf8Type}; +use crate::types::mapped::MappedScope; +use crate::types::value::{BinaryBytes, DataValue, Utf8Str, Utf8Type}; use crate::types::CharLengthUnits; use crate::types::LogicalType; use kite_sql_serde_macros::ReferenceSerialization; @@ -22,6 +23,7 @@ use ordered_float::OrderedFloat; use rust_decimal::Decimal; use std::fmt::Debug; use std::io::{Cursor, Read, Seek, SeekFrom, Write}; +use std::str; macro_rules! impl_tuple_value_serializable { ($name:ident, $variant:ident, $write_fn:expr, $read_fn:expr) => { @@ -38,11 +40,12 @@ macro_rules! impl_tuple_value_serializable { Ok(()) } - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, values: &mut Vec, _: &mut BytesPool, + _: Option<&MappedScope>, ) -> Result<(), DatabaseError> { values.push(DataValue::$variant(($read_fn)(reader)?)); Ok(()) @@ -53,11 +56,19 @@ macro_rules! impl_tuple_value_serializable { pub(crate) trait TupleValueSerializable: Debug { fn to_raw(&self, value: &DataValue, writer: &mut W) -> Result<(), DatabaseError>; - fn filling_value( + + /// Decodes one value. With a `scope` a string or binary points into the bytes under `reader`, + /// otherwise it is copied into `pool`'s buffers. + /// + /// # Safety + /// + /// With a `scope`, those bytes must be a read of the transaction holding it. + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, values: &mut Vec, pool: &mut BytesPool, + scope: Option<&MappedScope>, ) -> Result<(), DatabaseError>; } @@ -144,82 +155,83 @@ impl TupleValueSerializable for TupleValueSerializableImpl { } #[inline] - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, values: &mut Vec, pool: &mut BytesPool, + scope: Option<&MappedScope>, ) -> Result<(), DatabaseError> { match self { TupleValueSerializableImpl::Varbinary { len } => { - VarbinarySerializable { len: *len }.filling_value(reader, values, pool) + VarbinarySerializable { len: *len }.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::Boolean => { - BooleanSerializable.filling_value(reader, values, pool) + BooleanSerializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::Int8 => { - Int8Serializable.filling_value(reader, values, pool) + Int8Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::Int16 => { - Int16Serializable.filling_value(reader, values, pool) + Int16Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::Int32 => { - Int32Serializable.filling_value(reader, values, pool) + Int32Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::Int64 => { - Int64Serializable.filling_value(reader, values, pool) + Int64Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::UInt8 => { - UInt8Serializable.filling_value(reader, values, pool) + UInt8Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::UInt16 => { - UInt16Serializable.filling_value(reader, values, pool) + UInt16Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::UInt32 => { - UInt32Serializable.filling_value(reader, values, pool) + UInt32Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::UInt64 => { - UInt64Serializable.filling_value(reader, values, pool) + UInt64Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::Float32 => { - Float32Serializable.filling_value(reader, values, pool) + Float32Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::Float64 => { - Float64Serializable.filling_value(reader, values, pool) + Float64Serializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::Char { len, unit } => CharSerializable { len: *len, unit: *unit, } - .filling_value(reader, values, pool), + .filling_value(reader, values, pool, scope), TupleValueSerializableImpl::Varchar { len, unit } => VarcharSerializable { len: *len, unit: *unit, } - .filling_value(reader, values, pool), + .filling_value(reader, values, pool, scope), TupleValueSerializableImpl::Date => { - DateSerializable.filling_value(reader, values, pool) + DateSerializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::DateTime => { - DateTimeSerializable.filling_value(reader, values, pool) + DateTimeSerializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::Time { precision } => TimeSerializable { precision: *precision, } - .filling_value(reader, values, pool), + .filling_value(reader, values, pool, scope), TupleValueSerializableImpl::Timestamp { precision, zone } => TimeStampSerializable { precision: *precision, zone: *zone, } - .filling_value(reader, values, pool), + .filling_value(reader, values, pool, scope), TupleValueSerializableImpl::Decimal => { - DecimalSerializable.filling_value(reader, values, pool) + DecimalSerializable.filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::SkipFixed(len) => { - SkipFixed(*len).filling_value(reader, values, pool) + SkipFixed(*len).filling_value(reader, values, pool, scope) } TupleValueSerializableImpl::SkipVariable => { - SkipVariable.filling_value(reader, values, pool) + SkipVariable.filling_value(reader, values, pool, scope) } } } @@ -242,20 +254,47 @@ impl BytesPool { } } - pub(crate) fn string(&mut self, bytes: &[u8]) -> Result { - Ok(String::from_utf8(self.bytes(bytes))?) + /// # Safety + /// + /// With a `scope`, `bytes` must be a read of the transaction holding it. + pub(crate) unsafe fn string( + &mut self, + bytes: &[u8], + scope: Option<&MappedScope>, + ) -> Result { + Ok(match scope { + // SAFETY: the caller guarantees `bytes` is a read of `scope`'s transaction. + Some(scope) => unsafe { Utf8Str::mapped(str::from_utf8(bytes)?, scope) }, + None => String::from_utf8(self.bytes(bytes))?.into(), + }) + } + + /// # Safety + /// + /// With a `scope`, `bytes` must be a read of the transaction holding it. + pub(crate) unsafe fn binary( + &mut self, + bytes: &[u8], + scope: Option<&MappedScope>, + ) -> BinaryBytes { + match scope { + // SAFETY: the caller guarantees `bytes` is a read of `scope`'s transaction. + Some(scope) => unsafe { BinaryBytes::mapped(bytes, scope) }, + None => self.bytes(bytes).into(), + } } pub(crate) fn clone_value(&mut self, value: &DataValue) -> DataValue { match value { DataValue::Utf8 { value, ty, unit } => DataValue::Utf8 { - // SAFETY: the bytes are copied from a `str`. - value: unsafe { String::from_utf8_unchecked(self.bytes(value.as_bytes())) }, + // SAFETY: `bytes` returns a copy of what it is given. + value: unsafe { value.clone_with(|raw| self.bytes(raw)) }, ty: ty.clone(), unit: *unit, }, DataValue::Binary { value, len } => DataValue::Binary { - value: self.bytes(value), + // SAFETY: as above. + value: unsafe { value.clone_with(|raw| self.bytes(raw)) }, len: *len, }, value => value.clone(), @@ -264,11 +303,14 @@ impl BytesPool { pub(crate) fn recycle(&mut self, values: &mut Vec) { for value in values.drain(..) { - let mut buffer = match value { - DataValue::Utf8 { value, .. } => value.into_bytes(), - DataValue::Binary { value, .. } => value, + let buffer = match value { + DataValue::Utf8 { value, .. } => value.into_owned_raw(), + DataValue::Binary { value, .. } => value.into_owned_raw(), _ => continue, }; + let Some(mut buffer) = buffer else { + continue; + }; if self.0.len() < Self::CAPACITY && buffer.capacity() <= Self::MAX_BUFFER_LEN { buffer.clear(); self.0.push(buffer); @@ -496,11 +538,12 @@ impl TupleValueSerializable for CharSerializable { Ok(()) } - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, values: &mut Vec, pool: &mut BytesPool, + scope: Option<&MappedScope>, ) -> Result<(), DatabaseError> { // https://dev.mysql.com/doc/refman/8.0/en/char.html#:~:text=If%20a%20given%20value%20is%20stored%20into%20the%20CHAR(4)%20and%20VARCHAR(4)%20columns%2C%20the%20values%20retrieved%20from%20the%20columns%20are%20not%20always%20the%20same%20because%20trailing%20spaces%20are%20removed%20from%20CHAR%20columns%20upon%20retrieval.%20The%20following%20example%20illustrates%20this%20difference%3A let len = match self.unit { @@ -511,7 +554,8 @@ impl TupleValueSerializable for CharSerializable { let trimmed_len = bytes.iter().rposition(|&x| x != b' ').map_or(0, |i| i + 1); values.push(DataValue::Utf8 { - value: pool.string(&bytes[..trimmed_len])?, + // SAFETY: the caller's guarantee covers `bytes`. + value: unsafe { pool.string(&bytes[..trimmed_len], scope)? }, ty: Utf8Type::Fixed(self.len), unit: self.unit, }); @@ -536,15 +580,17 @@ impl TupleValueSerializable for VarcharSerializable { Ok(()) } - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, values: &mut Vec, pool: &mut BytesPool, + scope: Option<&MappedScope>, ) -> Result<(), DatabaseError> { let len = read_u32_le(reader)? as usize; values.push(DataValue::Utf8 { - value: pool.string(read_slice(reader, len)?)?, + // SAFETY: the caller's guarantee covers the bytes under `reader`. + value: unsafe { pool.string(read_slice(reader, len)?, scope)? }, ty: Utf8Type::Variable(self.len), unit: self.unit, }); @@ -562,15 +608,17 @@ impl TupleValueSerializable for VarbinarySerializable { Ok(()) } - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, values: &mut Vec, pool: &mut BytesPool, + scope: Option<&MappedScope>, ) -> Result<(), DatabaseError> { let len = read_u32_le(reader)? as usize; values.push(DataValue::Binary { - value: pool.bytes(read_slice(reader, len)?), + // SAFETY: the caller's guarantee covers the bytes under `reader`. + value: unsafe { pool.binary(read_slice(reader, len)?, scope) }, len: self.len, }); Ok(()) @@ -599,11 +647,12 @@ impl TupleValueSerializable for TimeSerializable { Ok(()) } - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, values: &mut Vec, _: &mut BytesPool, + _: Option<&MappedScope>, ) -> Result<(), DatabaseError> { let precision = self.precision.unwrap_or_default(); values.push(DataValue::Time32(read_u32_le(reader)?, precision)); @@ -620,11 +669,12 @@ impl TupleValueSerializable for TimeStampSerializable { Ok(()) } - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, values: &mut Vec, _: &mut BytesPool, + _: Option<&MappedScope>, ) -> Result<(), DatabaseError> { let precision = self.precision.unwrap_or_default(); values.push(DataValue::Time64( @@ -656,11 +706,12 @@ impl TupleValueSerializable for DecimalSerializable { )) } - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, _: &mut Vec, _: &mut BytesPool, + _: Option<&MappedScope>, ) -> Result<(), DatabaseError> { reader.seek(SeekFrom::Current(16))?; Err(DatabaseError::UnsupportedStmt( @@ -674,11 +725,12 @@ impl TupleValueSerializable for SkipFixed { unreachable!(); } - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, _: &mut Vec, _: &mut BytesPool, + _: Option<&MappedScope>, ) -> Result<(), DatabaseError> { reader.seek(SeekFrom::Current(self.0 as i64))?; Ok(()) @@ -690,11 +742,12 @@ impl TupleValueSerializable for SkipVariable { unreachable!(); } - fn filling_value( + unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, _: &mut Vec, _: &mut BytesPool, + _: Option<&MappedScope>, ) -> Result<(), DatabaseError> { let len = read_u32_le(reader)? as usize; reader.seek(SeekFrom::Current(len as i64))?; @@ -758,11 +811,15 @@ mod tests { let mut bytes = Vec::new(); serializer.to_raw(&value, &mut bytes)?; let mut values = Vec::new(); - serializer.filling_value( - &mut Cursor::new(bytes.as_slice()), - &mut values, - &mut BytesPool::default(), - )?; + // SAFETY: without a scope every value is copied. + unsafe { + serializer.filling_value( + &mut Cursor::new(bytes.as_slice()), + &mut values, + &mut BytesPool::default(), + None, + )? + }; assert_eq!(values.len(), 1); Ok(values.pop().unwrap()) } @@ -774,12 +831,13 @@ mod tests { #[test] fn bytes_pool_reuses_recycled_buffers() -> Result<(), DatabaseError> { let mut pool = BytesPool::default(); - let value = pool.string(b"abc")?; + // SAFETY: without a scope the string is copied. + let value = unsafe { pool.string(b"abc", None)? }; let buffer = value.as_ptr(); let large = vec![0; BytesPool::MAX_BUFFER_LEN + 1]; let mut values = vec![ DataValue::Binary { - value: large, + value: large.into(), len: None, }, DataValue::Utf8 { @@ -808,11 +866,15 @@ mod tests { unit: CharLengthUnits::Octets, }; let mut values = Vec::new(); - serializer.filling_value( - &mut Cursor::new(b"ab ".as_slice()), - &mut values, - &mut BytesPool::default(), - )?; + // SAFETY: without a scope every value is copied. + unsafe { + serializer.filling_value( + &mut Cursor::new(b"ab ".as_slice()), + &mut values, + &mut BytesPool::default(), + None, + )? + }; assert!(matches!(&values[..], [DataValue::Utf8 { value, .. }] if value == "ab")); Ok(()) } @@ -824,7 +886,7 @@ mod tests { ( TupleValueSerializableImpl::Varbinary { len: Some(8) }, DataValue::Binary { - value: vec![0, 255, 128], + value: vec![0, 255, 128].into(), len: Some(8), }, ), @@ -858,7 +920,7 @@ mod tests { unit: CharLengthUnits::Characters, }, DataValue::Utf8 { - value: "ab".to_string(), + value: "ab".into(), ty: Utf8Type::Fixed(4), unit: CharLengthUnits::Characters, }, @@ -869,7 +931,7 @@ mod tests { unit: CharLengthUnits::Octets, }, DataValue::Utf8 { - value: "ab".to_string(), + value: "ab".into(), ty: Utf8Type::Fixed(70), unit: CharLengthUnits::Octets, }, @@ -880,7 +942,7 @@ mod tests { unit: CharLengthUnits::Characters, }, DataValue::Utf8 { - value: "kite".to_string(), + value: "kite".into(), ty: Utf8Type::Variable(Some(8)), unit: CharLengthUnits::Characters, }, @@ -920,17 +982,24 @@ mod tests { serializer.to_raw(&value, &mut bytes)?; let mut values = Vec::new(); - serializer.filling_value( - &mut Cursor::new(bytes.as_slice()), - &mut values, - &mut BytesPool::default(), - )?; + // SAFETY: without a scope every value is copied. + unsafe { + serializer.filling_value( + &mut Cursor::new(bytes.as_slice()), + &mut values, + &mut BytesPool::default(), + None, + )? + }; assert_eq!(values, vec![value]); let skip_fixed = TupleValueSerializableImpl::SkipFixed(2); let mut values = vec![DataValue::Int32(1)]; let mut cursor = Cursor::new([1_u8, 2, 3].as_slice()); - skip_fixed.filling_value(&mut cursor, &mut values, &mut BytesPool::default())?; + // SAFETY: without a scope every value is copied. + unsafe { + skip_fixed.filling_value(&mut cursor, &mut values, &mut BytesPool::default(), None)? + }; assert_eq!(values, vec![DataValue::Int32(1)]); assert_eq!(cursor.position(), 2); @@ -940,7 +1009,15 @@ mod tests { bytes.extend_from_slice(&[1, 2, 3, 4]); let mut values = vec![DataValue::Int32(2)]; let mut cursor = Cursor::new(bytes.as_slice()); - skip_variable.filling_value(&mut cursor, &mut values, &mut BytesPool::default())?; + // SAFETY: without a scope every value is copied. + unsafe { + skip_variable.filling_value( + &mut cursor, + &mut values, + &mut BytesPool::default(), + None, + )? + }; assert_eq!(values, vec![DataValue::Int32(2)]); assert_eq!(cursor.position(), 7); diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 9583d3de..0a0fc9e6 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -16,6 +16,7 @@ use crate::catalog::{ColumnCatalog, ColumnRef}; use crate::errors::DatabaseError; use crate::iter_ext::Itertools; use crate::planner::MetaArena; +use crate::types::mapped::MappedScope; use crate::types::serialize::{BytesPool, TupleValueSerializable, TupleValueSerializableImpl}; use crate::types::value::DataValue; use std::borrow::Borrow; @@ -352,6 +353,21 @@ impl TupleImpl { self.pk_slot().as_ref() } + /// Copies every mapped string and binary so the tuple outlives its transaction. + pub(crate) fn materialize(&mut self) { + match self { + TupleImpl::Tuple(tuple) => { + tuple.pk.iter_mut().for_each(DataValue::materialize); + tuple.values.iter_mut().for_each(DataValue::materialize); + } + TupleImpl::Concat(concat) => { + concat.pk.iter_mut().for_each(DataValue::materialize); + concat.left.materialize(); + concat.right.materialize(); + } + } + } + /// The stored `Option` itself, for callers that encode it as a whole. #[inline] pub(crate) fn pk_slot(&self) -> &Option { @@ -731,13 +747,20 @@ impl Tuple { Tuple { pk, values } } + /// Decodes a stored row. With a `scope` strings point into `bytes`, otherwise they are + /// copied. + /// + /// # Safety + /// + /// With a `scope`, `bytes` must be a read of the transaction holding it. #[inline] - pub(crate) fn deserialize_from_into( + pub(crate) unsafe fn deserialize_from_into( &mut self, deserializers: I, bytes: &[u8], total_len: usize, pool: &mut BytesPool, + scope: Option<&MappedScope>, ) -> Result<(), DatabaseError> where I: IntoIterator, @@ -766,9 +789,12 @@ impl Tuple { } continue; } - deserializer - .borrow() - .filling_value(&mut cursor, &mut self.values, pool)?; + // SAFETY: the caller guarantees `bytes` is a read of `scope`'s transaction. + unsafe { + deserializer + .borrow() + .filling_value(&mut cursor, &mut self.values, pool, scope)? + }; } Ok(()) } @@ -960,7 +986,7 @@ mod tests { DataValue::Int32(0), DataValue::UInt32(1), DataValue::Utf8 { - value: "LOL".to_string(), + value: "LOL".into(), ty: Utf8Type::Variable(Some(2)), unit: CharLengthUnits::Characters, }, @@ -975,22 +1001,22 @@ mod tests { DataValue::Date32(0), DataValue::Decimal(Decimal::new(0, 3)), DataValue::Utf8 { - value: "K".to_string(), + value: "K".into(), ty: Utf8Type::Fixed(1), unit: CharLengthUnits::Characters, }, DataValue::Utf8 { - value: "LOL".to_string(), + value: "LOL".into(), ty: Utf8Type::Variable(Some(2)), unit: CharLengthUnits::Octets, }, DataValue::Utf8 { - value: "K".to_string(), + value: "K".into(), ty: Utf8Type::Fixed(10), unit: CharLengthUnits::Octets, }, DataValue::Utf8 { - value: "你".to_string(), + value: "你".into(), ty: Utf8Type::Fixed(3), unit: CharLengthUnits::Characters, }, @@ -1031,14 +1057,17 @@ mod tests { values: Vec::with_capacity(serializers.len()), }; tuples[0].serialize_to(&serializers, &mut bytes).unwrap(); - tuple_0 - .deserialize_from_into( + // SAFETY: without a scope strings are copied. + unsafe { + tuple_0.deserialize_from_into( &serializers, &bytes, columns.len(), &mut crate::types::serialize::BytesPool::default(), + None, ) - .unwrap(); + } + .unwrap(); assert_eq!(tuples[0], TupleImpl::from(tuple_0)); } @@ -1048,14 +1077,17 @@ mod tests { values: Vec::with_capacity(serializers.len()), }; tuples[1].serialize_to(&serializers, &mut bytes).unwrap(); - tuple_1 - .deserialize_from_into( + // SAFETY: without a scope strings are copied. + unsafe { + tuple_1.deserialize_from_into( &serializers, &bytes, columns.len(), &mut crate::types::serialize::BytesPool::default(), + None, ) - .unwrap(); + } + .unwrap(); assert_eq!(tuples[1], TupleImpl::from(tuple_1)); } @@ -1072,14 +1104,17 @@ mod tests { values: Vec::with_capacity(2), }; tuples[0].serialize_to(&serializers, &mut bytes).unwrap(); - tuple_2 - .deserialize_from_into( + // SAFETY: without a scope strings are copied. + unsafe { + tuple_2.deserialize_from_into( &projection_serializers, &bytes, columns.len(), &mut crate::types::serialize::BytesPool::default(), + None, ) - .unwrap(); + } + .unwrap(); assert_eq!( tuple_2, @@ -1093,14 +1128,17 @@ mod tests { let mut null_skipped = tuples[0].clone(); null_skipped[1] = DataValue::Null; null_skipped.serialize_to(&serializers, &mut bytes).unwrap(); - tuple_2 - .deserialize_from_into( + // SAFETY: without a scope strings are copied. + unsafe { + tuple_2.deserialize_from_into( &projection_serializers, &bytes, columns.len(), &mut crate::types::serialize::BytesPool::default(), + None, ) - .unwrap(); + } + .unwrap(); assert_eq!( tuple_2.values, @@ -1125,14 +1163,17 @@ mod tests { multi_pk_tuple .serialize_to(&serializers, &mut bytes) .unwrap(); - tuple_3 - .deserialize_from_into( + // SAFETY: without a scope strings are copied. + unsafe { + tuple_3.deserialize_from_into( &multiple_pk_serializers, &bytes, columns.len(), &mut crate::types::serialize::BytesPool::default(), + None, ) - .unwrap(); + } + .unwrap(); assert_eq!( tuple_3, @@ -1142,7 +1183,7 @@ mod tests { DataValue::Int32(0), DataValue::UInt32(1), DataValue::Utf8 { - value: "LOL".to_string(), + value: "LOL".into(), ty: Utf8Type::Variable(Some(2)), unit: CharLengthUnits::Characters, }, @@ -1273,14 +1314,17 @@ mod tests { .unwrap(); let mut tuple = Tuple::default(); - tuple - .deserialize_from_into( + // SAFETY: without a scope strings are copied. + unsafe { + tuple.deserialize_from_into( serializers.iter(), &bytes, 2, &mut crate::types::serialize::BytesPool::default(), + None, ) - .unwrap(); + } + .unwrap(); assert_eq!(tuple.values, vec![DataValue::Null, DataValue::Int32(7)]); assert_eq!( diff --git a/src/types/tuple_builder.rs b/src/types/tuple_builder.rs index 08cd6bc1..ba957232 100644 --- a/src/types/tuple_builder.rs +++ b/src/types/tuple_builder.rs @@ -33,7 +33,7 @@ impl<'a> TupleBuilder<'a> { pub fn build_result(message: String) -> Tuple { let values = vec![DataValue::Utf8 { - value: message, + value: message.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }]; @@ -45,7 +45,7 @@ impl<'a> TupleBuilder<'a> { tuple.pk = None; tuple.values.clear(); tuple.values.push(DataValue::Utf8 { - value: message, + value: message.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }); @@ -62,7 +62,7 @@ impl<'a> TupleBuilder<'a> { for (i, value) in row.into_iter().enumerate() { tuple.values.push( DataValue::Utf8 { - value: value.to_string(), + value: value.to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } diff --git a/src/types/value.rs b/src/types/value.rs index cb971338..e0cf810a 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -18,6 +18,7 @@ use crate::iter_ext::Itertools; use crate::planner::ScalarQueryRef; use crate::storage::table_codec::{BumpBytes, NOTNULL_TAG, NULL_TAG}; use crate::types::evaluator::cast::cast_create; +pub use crate::types::mapped::{BinaryBytes, Utf8Str}; use crate::types::CharLengthUnits; #[cfg(feature = "time")] use chrono::{ @@ -155,7 +156,7 @@ pub enum DataValue { UInt32(u32), UInt64(u64), Utf8 { - value: String, + value: Utf8Str, ty: Utf8Type, unit: CharLengthUnits, }, @@ -169,7 +170,7 @@ pub enum DataValue { Decimal(Decimal), Tuple(Vec), Binary { - value: Vec, + value: BinaryBytes, len: Option, }, } @@ -759,7 +760,7 @@ impl DataValue { pub fn new_utf8(string: String) -> Self { DataValue::Utf8 { ty: Utf8Type::Fixed(string.len() as u32), - value: string, + value: string.into(), unit: CharLengthUnits::Characters, } } @@ -867,6 +868,16 @@ impl DataValue { ) } + /// Copies every mapped string and binary so the value outlives its transaction. + pub(crate) fn materialize(&mut self) { + match self { + DataValue::Utf8 { value, .. } => value.materialize(), + DataValue::Binary { value, .. } => value.materialize(), + DataValue::Tuple(values) => values.iter_mut().for_each(DataValue::materialize), + _ => (), + } + } + #[inline] pub fn is_null(&self) -> bool { matches!(self, DataValue::Null) @@ -917,17 +928,17 @@ impl DataValue { LogicalType::Float => DataValue::Float32(OrderedFloat(0.0)), LogicalType::Double => DataValue::Float64(OrderedFloat(0.0)), LogicalType::Char(len, unit) => DataValue::Utf8 { - value: String::new(), + value: String::new().into(), ty: Utf8Type::Fixed(*len), unit: *unit, }, LogicalType::Varchar(len, unit) => DataValue::Utf8 { - value: String::new(), + value: String::new().into(), ty: Utf8Type::Variable(*len), unit: *unit, }, LogicalType::Varbinary(len) => DataValue::Binary { - value: Vec::new(), + value: Vec::new().into(), len: *len, }, LogicalType::Date => { @@ -1384,16 +1395,16 @@ impl DataValue { Ok(DataValue::Boolean(b[0] == b'1')) } LogicalType::Varbinary(len) => Ok(DataValue::Binary { - value: Self::decode_string(reader)?, + value: Self::decode_string(reader)?.into(), len: *len, }), LogicalType::Varchar(len, unit) => Ok(DataValue::Utf8 { - value: String::from_utf8(Self::decode_string(reader)?)?, + value: String::from_utf8(Self::decode_string(reader)?)?.into(), ty: Utf8Type::Variable(*len), unit: *unit, }), LogicalType::Char(len, unit) => Ok(DataValue::Utf8 { - value: String::from_utf8(Self::decode_string(reader)?)?, + value: String::from_utf8(Self::decode_string(reader)?)?.into(), ty: Utf8Type::Fixed(*len), unit: *unit, }), @@ -1838,14 +1849,17 @@ impl From> for DataValue { impl From> for DataValue { fn from(value: Vec) -> Self { - Self::Binary { value, len: None } + Self::Binary { + value: value.into(), + len: None, + } } } impl From for DataValue { fn from(value: String) -> Self { DataValue::Utf8 { - value, + value: value.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } @@ -1856,7 +1870,7 @@ impl From> for DataValue { fn from(value: Option) -> Self { if let Some(value) = value { DataValue::Utf8 { - value, + value: value.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } @@ -1957,7 +1971,7 @@ impl fmt::Display for DataValue { DataValue::UInt64(e) => write!(f, "{e}")?, DataValue::Binary { value, .. } => { write!(f, "X'")?; - for byte in value { + for byte in value.iter() { write!(f, "{byte:02X}")?; } write!(f, "'")?; @@ -2067,7 +2081,7 @@ mod test { #[test] fn memcomparable_len_hint_matches_encoding() -> Result<(), DatabaseError> { let utf8 = |value: &str| DataValue::Utf8 { - value: value.to_string(), + value: value.to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }; @@ -2091,7 +2105,7 @@ mod test { utf8(""), utf8("kite_sql"), DataValue::Binary { - value: vec![1; 9], + value: vec![1; 9].into(), len: None, }, DataValue::Tuple(vec![DataValue::Int32(1), DataValue::Null, utf8("a")]), @@ -2206,7 +2220,7 @@ mod test { fn utf8(value: &str) -> DataValue { DataValue::Utf8 { - value: value.to_string(), + value: value.to_string().into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } @@ -2214,7 +2228,7 @@ mod test { fn fixed_utf8(value: &str, len: u32, unit: CharLengthUnits) -> DataValue { DataValue::Utf8 { - value: value.to_string(), + value: value.to_string().into(), ty: Utf8Type::Fixed(len), unit, } @@ -2249,7 +2263,7 @@ mod test { assert_eq!( DataValue::new_utf8("kite".to_string()), DataValue::Utf8 { - value: "kite".to_string(), + value: "kite".into(), ty: Utf8Type::Fixed(4), unit: CharLengthUnits::Characters, } @@ -2360,7 +2374,7 @@ mod test { ( LogicalType::Varchar(Some(3), CharLengthUnits::Octets), DataValue::Utf8 { - value: String::new(), + value: String::new().into(), ty: Utf8Type::Variable(Some(3)), unit: CharLengthUnits::Octets, }, @@ -2553,7 +2567,7 @@ mod test { for length in [0, 7, 8, 9, 65] { cases.push(( DataValue::Binary { - value: vec![0xff; length], + value: vec![0xff; length].into(), len: None, }, LogicalType::Varbinary(None), @@ -2864,7 +2878,7 @@ mod test { assert_eq!( DataValue::from(Some("abc".to_string())), DataValue::Utf8 { - value: "abc".to_string(), + value: "abc".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, } @@ -3633,25 +3647,25 @@ mod test { let v_null = DataValue::Null; let v_a = DataValue::Utf8 { - value: "a".to_string(), + value: "a".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }; let v_ab = DataValue::Utf8 { - value: "ab".to_string(), + value: "ab".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }; let v_b = DataValue::Utf8 { - value: "b".to_string(), + value: "b".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }; let v_zh = DataValue::Utf8 { - value: "中".to_string(), + value: "中".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }; diff --git a/src/wasm.rs b/src/wasm.rs index c4123ea5..6090c285 100644 --- a/src/wasm.rs +++ b/src/wasm.rs @@ -48,7 +48,7 @@ fn data_value_to_js(value: &DataValue) -> Result { DataValue::UInt16(value) => Ok(JsValue::from_f64(*value as f64)), DataValue::UInt32(value) => Ok(JsValue::from_f64(*value as f64)), DataValue::UInt64(value) => Ok(JsValue::from_f64(*value as f64)), - DataValue::Binary { value, .. } => Ok(Uint8Array::from(value.as_slice()).into()), + DataValue::Binary { value, .. } => Ok(Uint8Array::from(value.as_bytes()).into()), DataValue::Utf8 { value, ty, unit } => { let object = Object::new(); set_prop(&object, "value", JsValue::from_str(value))?; diff --git a/tests/macros-test/src/main.rs b/tests/macros-test/src/main.rs index 2cf5f70e..475151bd 100644 --- a/tests/macros-test/src/main.rs +++ b/tests/macros-test/src/main.rs @@ -18,7 +18,7 @@ fn main() {} #[cfg(test)] mod test { use kite_sql::catalog::column::{ColumnCatalog, ColumnDesc}; - use kite_sql::db::{DataBaseBuilder, Database, ResultIter}; + use kite_sql::db::{DataBaseBuilder, Database, ResultIter, ResultRow, TxnResultIter}; use kite_sql::errors::DatabaseError; use kite_sql::expression::function::scala::ScalarFunctionImpl; use kite_sql::expression::function::table::TableFunctionImpl; @@ -59,7 +59,7 @@ mod test { let values = vec![ DataValue::Int32(9), DataValue::Utf8 { - value: "LOL".to_string(), + value: "LOL".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }, @@ -98,7 +98,7 @@ mod test { #[test] fn test_from_data_value_reports_conversion_error() { let err = ::from_data_value(DataValue::Utf8 { - value: "not an integer".to_string(), + value: "not an integer".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }) @@ -242,10 +242,10 @@ mod test { } #[derive(Default, Debug, PartialEq, Projection)] - struct UserSummary { + struct UserSummary<'a> { id: i32, #[projection(rename = "user_name")] - display_name: String, + display_name: Cow<'a, str>, age: Option, } @@ -332,7 +332,7 @@ mod test { }, c2: String => |inner: &mut MyStruct, value| { if let DataValue::Utf8 { value, .. } = value { - inner.c2 = value; + inner.c2 = value.into_string(); } } ) @@ -344,9 +344,11 @@ mod test { let mut plan_arena = PlanArena::new(&table_arena); let (tuple, schema) = build_tuple(&mut plan_arena); let schema = SchemaView::new(&schema, &plan_arena); - let my_struct = - ::from_query_row(&schema, &mut tuple.into()) - .unwrap(); + let my_struct = ::from_query_row( + &schema, + &mut ResultRow::new(&mut tuple.into()), + ) + .unwrap(); println!("{:?}", my_struct); @@ -377,7 +379,7 @@ mod test { vec![ DataValue::Int32(9), DataValue::Utf8 { - value: "LOL".to_string(), + value: "LOL".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }, @@ -388,7 +390,7 @@ mod test { let schema = SchemaView::new(&schema, &plan_arena); let derived = ::from_query_row( &schema, - &mut tuple.into(), + &mut ResultRow::new(&mut tuple.into()), ) .unwrap(); @@ -1093,22 +1095,38 @@ mod test { vec![ UserSummary { id: 1, - display_name: "Alice".to_string(), + display_name: "Alice".into(), age: Some(18), }, UserSummary { id: 2, - display_name: "Bob".to_string(), + display_name: "Bob".into(), age: Some(30), }, UserSummary { id: 3, - display_name: "A'lex".to_string(), + display_name: "A'lex".into(), age: None, }, ] ); + // The same projection decodes from a transaction's rows too, where it may borrow its + // strings from the transaction. + let mut tx = database.new_transaction(true)?; + let borrowed_users = tx + .bind(|ctx| { + ctx.from::()? + .project::>()? + .order_by(User::id())? + .finish() + })? + .txn_orm::>() + .collect::, _>>()?; + assert_eq!(borrowed_users, projected_users); + drop(borrowed_users); + tx.commit()?; + let projected_user = database .bind(|ctx| { ctx.from::()? @@ -1123,7 +1141,7 @@ mod test { projected_user, Some(UserSummary { id: 1, - display_name: "Alice".to_string(), + display_name: "Alice".into(), age: Some(18), }) ); @@ -1306,7 +1324,7 @@ mod test { aliased_projection, Some(UserSummary { id: 2, - display_name: "Bob".to_string(), + display_name: "Bob".into(), age: Some(30), }) ); @@ -2022,7 +2040,7 @@ mod test { union_projection, Some(UserSummary { id: 2, - display_name: "Bob".to_string(), + display_name: "Bob".into(), age: Some(30), }) ); @@ -2053,7 +2071,7 @@ mod test { ordered_union_projection, Some(UserSummary { id: 2, - display_name: "Bob".to_string(), + display_name: "Bob".into(), age: Some(30), }) ); @@ -3043,7 +3061,7 @@ mod test { &[ arena.alloc_expression(ScalarExpression::Constant(DataValue::Int8(1))), arena.alloc_expression(ScalarExpression::Constant(DataValue::Utf8 { - value: "1".to_string(), + value: "1".into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, })), @@ -3057,7 +3075,7 @@ mod test { assert_eq!( function.summary, FunctionSummary { - name: "sum".to_string().into(), + name: "sum".into(), arg_types: vec![LogicalType::Integer, LogicalType::Integer], } ); @@ -3080,7 +3098,7 @@ mod test { assert_eq!( function.summary, FunctionSummary { - name: "test_numbers".to_string().into(), + name: "test_numbers".into(), arg_types: vec![LogicalType::Integer], } ); diff --git a/tpcc/src/backend/sqlite.rs b/tpcc/src/backend/sqlite.rs index ae72fd92..a1478159 100644 --- a/tpcc/src/backend/sqlite.rs +++ b/tpcc/src/backend/sqlite.rs @@ -239,8 +239,8 @@ fn convert_value(value: &DataValue) -> Result { DataValue::UInt16(v) => Value::Integer(*v as i64), DataValue::UInt32(v) => Value::Integer(*v as i64), DataValue::UInt64(v) => Value::Integer(*v as i64), - DataValue::Binary { value, .. } => Value::Binary(value.clone()), - DataValue::Utf8 { value, .. } => Value::String(value.clone()), + DataValue::Binary { value, .. } => Value::Binary(value.to_vec()), + DataValue::Utf8 { value, .. } => Value::String(value.to_string()), DataValue::Date32(v) => Value::Integer(*v as i64), DataValue::Date64(v) => Value::String( Utc.timestamp_opt(*v, 0) @@ -369,7 +369,7 @@ fn convert_statement_row( LogicalType::Bigint => DataValue::Int64(statement.read::(idx)?), LogicalType::Decimal(..) => DataValue::Decimal(read_decimal(statement, idx)?), LogicalType::Varchar(..) | LogicalType::Char(..) => DataValue::Utf8 { - value: statement.read::(idx)?, + value: statement.read::(idx)?.into(), ty: Utf8Type::Variable(None), unit: CharLengthUnits::Characters, }, From 6d42021ccd8797c3a708ccca19bc0dca53e848c2 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 4 Nov 2026 13:15:42 +0800 Subject: [PATCH 03/14] perf(storage): reuse bound and parameter buffers across executions - StorageTransaction::range/range_rev and remove_range take a BytesPool; LMDB and memory iterators copy their bounds into pooled buffers and InnerIter::recycle hands them back. - TableCodec range-bound helpers hand their BytesPool to the callback. - IndexIter takes its encode buffers from the pool lazily, and finished executions recycle every storage iterator's buffers. - DataValue implements clone_from to reuse string, binary and tuple buffers; ParamArena::bind overwrites the previous values with it. --- src/db.rs | 11 +- src/execution/ddl/mod.rs | 2 +- src/execution/dml/analyze.rs | 31 +++--- src/execution/mod.rs | 28 ++++- src/planner/arena.rs | 40 ++++--- src/storage/lmdb.rs | 39 +++++-- src/storage/memory.rs | 25 ++++- src/storage/mod.rs | 195 ++++++++++++++++++++++------------- src/storage/rocksdb.rs | 11 +- src/storage/table_codec.rs | 158 ++++++++++++++-------------- src/types/mapped.rs | 7 ++ src/types/serialize.rs | 25 +++-- src/types/value.rs | 75 +++++++++++++- 13 files changed, 437 insertions(+), 210 deletions(-) diff --git a/src/db.rs b/src/db.rs index 49e09010..38a626a8 100644 --- a/src/db.rs +++ b/src/db.rs @@ -54,6 +54,7 @@ use crate::storage::{ CheckpointableStorage, NameMapping, StatisticsMetaCache, Storage, StorageTransaction, TableCache, TransactionIsolationLevel, ViewCache, }; +use crate::types::serialize::BytesPool; use crate::types::tuple::{Schema, SchemaView, TupleImpl}; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -1562,12 +1563,12 @@ struct TransactionGuard<'a, 'p, T: StorageTransaction + 'p> { impl TransactionGuard<'_, '_, T> { #[inline] - fn finish(&mut self) { + fn finish(&mut self, pool: Option<&mut BytesPool>) { if !self.active { return; } self.active = false; - self.arena.release_iters(); + self.arena.release_iters(pool); } } @@ -1595,11 +1596,11 @@ impl<'p, T: StorageTransaction + 'p> TransactionGuard<'_, 'p, T> { ))) } Ok(false) => { - self.finish(); + self.finish(Some(executor.buffer.bytes_pool())); Ok(None) } Err(err) => { - self.finish(); + self.finish(Some(executor.buffer.bytes_pool())); Err(err) } } @@ -1608,7 +1609,7 @@ impl<'p, T: StorageTransaction + 'p> TransactionGuard<'_, 'p, T> { impl Drop for TransactionGuard<'_, '_, T> { fn drop(&mut self) { - self.finish(); + self.finish(None); } } diff --git a/src/execution/ddl/mod.rs b/src/execution/ddl/mod.rs index 8754ce2d..ff941ae5 100644 --- a/src/execution/ddl/mod.rs +++ b/src/execution/ddl/mod.rs @@ -59,7 +59,7 @@ fn read_tuple_batch( Bound::Excluded(bytes) => Bound::Excluded(bytes.as_slice()), Bound::Unbounded => Bound::Included(min), }; - let mut iter = transaction.range(lower, Bound::Included(max))?; + let mut iter = transaction.range(lower, Bound::Included(max), pool)?; let mut len = 0; while len < batch_size { diff --git a/src/execution/dml/analyze.rs b/src/execution/dml/analyze.rs index d3d2a293..4f887f06 100644 --- a/src/execution/dml/analyze.rs +++ b/src/execution/dml/analyze.rs @@ -597,14 +597,16 @@ mod test { ); let mut transaction = kite_sql.storage.transaction()?; let mut table_codec = TableCodec::default(); - let keys: Vec> = table_codec.with_index_statistics_bound(0, 1, |min, max| { - let mut iter = transaction.range(Bound::Included(min), Bound::Included(max))?; - let mut keys = Vec::new(); - while let Some((key, _)) = iter.try_next()? { - keys.push(key.to_vec()); - } - Ok(keys) - })?; + let keys: Vec> = + table_codec.with_index_statistics_bound(0, 1, |min, max, pool| { + let mut iter = + transaction.range(Bound::Included(min), Bound::Included(max), pool)?; + let mut keys = Vec::new(); + while let Some((key, _)) = iter.try_next()? { + keys.push(key.to_vec()); + } + Ok(keys) + })?; for key in keys { transaction.remove(&key)?; } @@ -669,8 +671,9 @@ mod test { let count = { let transaction = kite_sql.storage.transaction()?; let mut table_codec = TableCodec::default(); - table_codec.with_statistics_bound(0, |min, max| { - let mut iter = transaction.range(Bound::Included(min), Bound::Included(max))?; + table_codec.with_statistics_bound(0, |min, max, pool| { + let mut iter = + transaction.range(Bound::Included(min), Bound::Included(max), pool)?; let mut count = 0; while iter.try_next()?.is_some() { count += 1; @@ -685,8 +688,8 @@ mod test { let transaction = kite_sql.storage.transaction()?; let mut table_codec = TableCodec::default(); - let keys = table_codec.with_statistics_bound(0, |min, max| { - let mut iter = transaction.range(Bound::Included(min), Bound::Included(max))?; + let keys = table_codec.with_statistics_bound(0, |min, max, pool| { + let mut iter = transaction.range(Bound::Included(min), Bound::Included(max), pool)?; let mut keys = 0; while iter.try_next()?.is_some() { keys += 1; @@ -714,9 +717,9 @@ mod test { assert!(kite_sql.state.meta_cache().columns.is_empty()); assert!(kite_sql.state.meta_cache().indexes.is_empty()); let transaction = kite_sql.storage.transaction()?; - table_codec.with_statistics_bound(0, |min, max| { + table_codec.with_statistics_bound(0, |min, max, pool| { assert!(transaction - .range(Bound::Included(min), Bound::Included(max))? + .range(Bound::Included(min), Bound::Included(max), pool)? .try_next()? .is_none()); Ok(()) diff --git a/src/execution/mod.rs b/src/execution/mod.rs index 7204af28..9eaf883a 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -78,7 +78,9 @@ use crate::planner::operator::{Operator, PhysicalOption, PlanImpl}; use crate::planner::MetaArena; use crate::planner::{ExecMetaArena, ExprRef, LogicalPlan, PlanArena, PlanKeeper}; use crate::storage::table_codec::TableCodec; -use crate::storage::{IndexIter, StorageTransaction, TableCache, TableIter, TupleIter, ViewIter}; +use crate::storage::{ + IndexIter, InnerIter, StorageTransaction, TableCache, TableIter, TupleIter, ViewIter, +}; use crate::types::index::RuntimeIndexProbe; use crate::types::serialize::BytesPool; use crate::types::tuple::{ConcatPool, Input, Schema, Tuple, TupleId, TupleImpl}; @@ -610,6 +612,16 @@ impl<'t, T: StorageTransaction + 't> StorageIter<'t, T> { matches!(self, StorageIter::Empty) } + fn recycle(self, pool: &mut BytesPool) { + match self { + StorageIter::Tuple(iter) => iter.recycle(pool), + StorageIter::Index(iter) => iter.recycle(pool), + StorageIter::Table(iter) => iter.iter.recycle(pool), + StorageIter::View(iter) => iter.iter.recycle(pool), + StorageIter::Empty => (), + } + } + pub(crate) fn as_tuple(&mut self) -> &mut TupleIter<'t, T> { match self { StorageIter::Tuple(iter) => iter, @@ -699,6 +711,10 @@ impl ExecBuffer { pub(crate) fn result_mut(&mut self) -> &mut TupleImpl { &mut self.result.tuple } + + pub(crate) fn bytes_pool(&mut self) -> &mut BytesPool { + self.table_codec.bytes_pool() + } } impl<'a, 't, T: StorageTransaction + 'a> ExecArena<'a, 't, T> { @@ -718,8 +734,14 @@ impl<'a, 't, T: StorageTransaction + 'a> ExecArena<'a, 't, T> { self } - pub(crate) fn release_iters(&mut self) { - self.iters.fill_with(|| StorageIter::Empty); + /// Drops every storage iterator, handing their buffers back to `pool` when there is one. + pub(crate) fn release_iters(&mut self, mut pool: Option<&mut BytesPool>) { + for iter in &mut self.iters { + let iter = mem::replace(iter, StorageIter::Empty); + if let Some(pool) = pool.as_deref_mut() { + iter.recycle(pool); + } + } } pub(crate) fn take_ddl_apply(&mut self) -> Vec { diff --git a/src/planner/arena.rs b/src/planner/arena.rs index 4496adb0..3015febd 100644 --- a/src/planner/arena.rs +++ b/src/planner/arena.rs @@ -31,6 +31,7 @@ use crate::types::LogicalType; use std::cell::UnsafeCell; use std::collections::HashSet; use std::fmt; +use std::mem; struct ScalarQueryRelocator { source_arena: usize, @@ -1218,26 +1219,31 @@ impl<'a> ParamArena<'a> { unreachable!("parameter expression must be a single parameter constant"); }; let id = *id; - let value = params + let source = params .iter() .find_map(|(candidate, value)| (*candidate == id).then_some(value)) - .ok_or_else(|| DatabaseError::parameter_not_found(format!("${id}")))? - .clone() - .cast(ty)?; - // reuse - match self.expressions.get_mut(slot) { - Some(( - _, - ArenaExpr { - expression: ScalarExpression::Constant(bound), - param: None, - }, - )) => *bound = value, + .ok_or_else(|| DatabaseError::parameter_not_found(format!("${id}")))?; + let constant = || ArenaExpr::new(ScalarExpression::Constant(DataValue::Null)); + if self.expressions.len() == slot { + self.expressions.push((Some(id), constant())); + } + let (_, expr) = &mut self.expressions[slot]; + if !matches!( + expr, + ArenaExpr { + expression: ScalarExpression::Constant(_), + param: None, + } + ) { // An execution rewrote the slot; rebuild it. - Some((_, expr)) => *expr = ArenaExpr::new(ScalarExpression::Constant(value)), - None => self - .expressions - .push((Some(id), ArenaExpr::new(ScalarExpression::Constant(value)))), + *expr = constant(); + } + let ScalarExpression::Constant(value) = &mut expr.expression else { + unreachable!("the slot was just made a constant"); + }; + value.clone_from(source); + if value.logical_type() != *ty { + *value = mem::replace(value, DataValue::Null).cast(ty)?; } } Ok(()) diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs index 3aa5dfb8..7e649278 100644 --- a/src/storage/lmdb.rs +++ b/src/storage/lmdb.rs @@ -15,10 +15,11 @@ use crate::errors::DatabaseError; use crate::storage::table_codec::Bytes; use crate::storage::{ - bounds_contain, bytes_bound_as_slice, owned_bound, InnerIter, KeyValueRef, Storage, - StorageTransaction, TransactionIsolationLevel, + bounds_contain, bytes_bound_as_slice, InnerIter, KeyValueRef, Storage, StorageTransaction, + TransactionIsolationLevel, }; use crate::types::mapped::MappedScope; +use crate::types::serialize::BytesPool; use lmdb::{ Cursor, Database, DatabaseFlags, Environment, EnvironmentFlags, RoCursor, RoTransaction, RwTransaction, Transaction as _, WriteFlags, @@ -239,14 +240,24 @@ enum Step { type BytesRange = (Bound, Bound); impl Scan { - fn new(min: Bound<&[u8]>, max: Bound<&[u8]>, reverse: bool) -> Self { + fn new(min: Bound<&[u8]>, max: Bound<&[u8]>, reverse: bool, pool: &mut BytesPool) -> Self { + let mut owned = |bound: Bound<&[u8]>| bound.map(|key| pool.bytes(key)); Self { step: Step::Seek, - range: (owned_bound(min), owned_bound(max)), + range: (owned(min), owned(max)), reverse, } } + fn recycle(self, pool: &mut BytesPool) { + let (min, max) = self.range; + for bound in [min, max] { + if let Bound::Included(bytes) | Bound::Excluded(bytes) = bound { + pool.recycle_buffer(bytes); + } + } + } + fn start(&self) -> Bound<&[u8]> { if self.reverse { bytes_bound_as_slice(&self.range.1) @@ -313,6 +324,10 @@ impl InnerIter for LmdbIter<'_> { fn try_next(&mut self) -> Result>, DatabaseError> { self.scan.next(&self.cursor) } + + fn recycle(self, pool: &mut BytesPool) { + self.scan.recycle(pool); + } } fn cursor_get<'txn, C: Cursor<'txn>>( @@ -409,10 +424,11 @@ impl StorageTransaction for LmdbTransaction<'_> { &'txn self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, + pool: &mut BytesPool, ) -> Result, DatabaseError> { Ok(LmdbIter { cursor: self.tx.open_ro_cursor(self.db)?, - scan: Scan::new(min, max, false), + scan: Scan::new(min, max, false, pool), }) } @@ -420,19 +436,26 @@ impl StorageTransaction for LmdbTransaction<'_> { &'txn self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, + pool: &mut BytesPool, ) -> Result, DatabaseError> { Ok(LmdbIter { cursor: self.tx.open_ro_cursor(self.db)?, - scan: Scan::new(min, max, true), + scan: Scan::new(min, max, true, pool), }) } - fn remove_range(&mut self, min: Bound<&[u8]>, max: Bound<&[u8]>) -> Result<(), DatabaseError> { + fn remove_range( + &mut self, + min: Bound<&[u8]>, + max: Bound<&[u8]>, + pool: &mut BytesPool, + ) -> Result<(), DatabaseError> { let mut cursor = self.tx.writable()?.open_rw_cursor(self.db)?; - let mut scan = Scan::new(min, max, false); + let mut scan = Scan::new(min, max, false, pool); while scan.next(&cursor)?.is_some() { cursor.del(WriteFlags::empty())?; } + scan.recycle(pool); Ok(()) } diff --git a/src/storage/memory.rs b/src/storage/memory.rs index 8acd391b..3ca4e331 100644 --- a/src/storage/memory.rs +++ b/src/storage/memory.rs @@ -15,9 +15,10 @@ use crate::errors::DatabaseError; use crate::storage::table_codec::Bytes; use crate::storage::{ - bytes_bound_as_slice, owned_bound, EmptyStorageMetrics, InnerIter, Storage, StorageTransaction, + bytes_bound_as_slice, EmptyStorageMetrics, InnerIter, Storage, StorageTransaction, TransactionIsolationLevel, }; +use crate::types::serialize::BytesPool; use std::cell::{Cell, Ref, RefCell}; use std::collections::{BTreeMap, Bound}; use std::rc::Rc; @@ -89,11 +90,13 @@ impl<'txn> MemoryIter<'txn> { min: Bound<&[u8]>, max: Bound<&[u8]>, reverse: bool, + pool: &mut BytesPool, ) -> Self { + let mut owned = |bound: Bound<&[u8]>| bound.map(|key| pool.bytes(key)); Self { map, position: Position::Start, - range: (owned_bound(min), owned_bound(max)), + range: (owned(min), owned(max)), reverse, } } @@ -142,6 +145,18 @@ impl InnerIter for MemoryIter<'_> { // unmodified for as long as the slice is used (the same contract as LMDB cursors). Ok(Some((key.as_slice(), unsafe { &*value }))) } + + fn recycle(self, pool: &mut BytesPool) { + let (min, max) = self.range; + for bound in [min, max] { + if let Bound::Included(bytes) | Bound::Excluded(bytes) = bound { + pool.recycle_buffer(bytes); + } + } + if let Position::After(bytes) = self.position { + pool.recycle_buffer(bytes); + } + } } impl StorageTransaction for MemoryTransaction { @@ -201,16 +216,18 @@ impl StorageTransaction for MemoryTransaction { &'txn self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, + pool: &mut BytesPool, ) -> Result, DatabaseError> { - Ok(MemoryIter::new(&self.inner, min, max, false)) + Ok(MemoryIter::new(&self.inner, min, max, false, pool)) } fn range_rev<'txn, 'key>( &'txn self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, + pool: &mut BytesPool, ) -> Result, DatabaseError> { - Ok(MemoryIter::new(&self.inner, min, max, true)) + Ok(MemoryIter::new(&self.inner, min, max, true, pool)) } fn commit(self) -> Result<(), DatabaseError> { diff --git a/src/storage/mod.rs b/src/storage/mod.rs index fdaae9aa..88b58c08 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -36,7 +36,7 @@ use crate::planner::{MetaArena, TableArenaCell}; use crate::storage::table_codec::{Bytes, IndexStatisticsCodecType, TableCodec, BOUND_MAX_TAG}; use crate::types::index::{Index, IndexId, IndexMeta, IndexMetaRef, IndexType, RuntimeIndexProbe}; use crate::types::mapped::MappedScope; -use crate::types::serialize::TupleValueSerializableImpl; +use crate::types::serialize::{BytesPool, TupleValueSerializableImpl}; use crate::types::tuple::{Tuple, TupleId, TupleImpl, TupleLike}; use crate::types::value::{DataValue, OrderedMapping, TupleMappingRef}; use crate::types::{ColumnId, LogicalType}; @@ -242,8 +242,8 @@ pub trait StorageTransaction: Sized { ) -> Result, DatabaseError> { let offset = bounds.0.unwrap_or(0); - table_codec.with_tuple_bound(table_id, |min, max, _| { - let iter = self.range(Bound::Included(min), Bound::Included(max))?; + table_codec.with_tuple_bound(table_id, |min, max, pool| { + let iter = self.range(Bound::Included(min), Bound::Included(max), pool)?; Ok(TupleIter { table_id, @@ -425,12 +425,12 @@ pub trait StorageTransaction: Sized { table: &TableCatalog, ) -> Result<(), DatabaseError> { let table_id = table.id(); - table_codec.with_columns_bound(table_id, |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_columns_bound(table_id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; - table_codec.with_index_meta_bound(table_id, |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_index_meta_bound(table_id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; for column in table.columns().map(|column| arena.column(*column)) { @@ -541,8 +541,8 @@ pub trait StorageTransaction: Sized { plan_arena, )?; self.rewrite_table_metadata(table_codec, plan_arena, &updated_table)?; - table_codec.with_statistics_bound(updated_table.id(), |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_statistics_bound(updated_table.id(), |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; Ok(updated_table) @@ -629,8 +629,8 @@ pub trait StorageTransaction: Sized { self.remove(key) })?; - table_codec.with_index_bound(table_id, index_meta.id, |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_index_bound(table_id, index_meta.id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; self.remove_index_statistics(table_codec, table_id, index_meta.id)?; @@ -792,8 +792,8 @@ pub trait StorageTransaction: Sized { self.remove(key) })?; - table_codec.with_index_bound(table.id(), index_id, |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_index_bound(table.id(), index_id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; self.remove_index_statistics(table_codec, table.id(), index_id)?; @@ -823,12 +823,12 @@ pub trait StorageTransaction: Sized { }; self.drop_data(table_codec, meta.id)?; - table_codec.with_columns_bound(meta.id, |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_columns_bound(meta.id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; - table_codec.with_index_meta_bound(meta.id, |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_index_meta_bound(meta.id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; table_codec.with_root_table(table_name.as_ref(), None, plan_arena, |key, _| { @@ -843,16 +843,16 @@ pub trait StorageTransaction: Sized { table_codec: &mut TableCodec, table_id: TableId, ) -> Result<(), DatabaseError> { - table_codec.with_tuple_bound(table_id, |min, max, _| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_tuple_bound(table_id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; - table_codec.with_all_index_bound(table_id, |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_all_index_bound(table_id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; - table_codec.with_statistics_bound(table_id, |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_statistics_bound(table_id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) }) } @@ -902,9 +902,9 @@ pub trait StorageTransaction: Sized { &'a self, table_codec: &mut TableCodec, ) -> Result, DatabaseError> { - table_codec.with_view_bound(|min, max| { + table_codec.with_view_bound(|min, max, pool| { Ok(ViewIter { - iter: self.range(Bound::Included(min), Bound::Included(max))?, + iter: self.range(Bound::Included(min), Bound::Included(max), pool)?, }) }) } @@ -947,9 +947,9 @@ pub trait StorageTransaction: Sized { &'a self, table_codec: &mut TableCodec, ) -> Result, DatabaseError> { - table_codec.with_root_table_bound(|min, max| { + table_codec.with_root_table_bound(|min, max, pool| { Ok(TableIter { - iter: self.range(Bound::Included(min), Bound::Included(max))?, + iter: self.range(Bound::Included(min), Bound::Included(max), pool)?, }) }) } @@ -1058,8 +1058,8 @@ pub trait StorageTransaction: Sized { index_id: IndexId, arena: &mut (impl MetaArena + ?Sized), ) -> Result, DatabaseError> { - table_codec.with_index_statistics_bound(table_id, index_id, |min, max| { - let mut iter = self.range(Bound::Included(min), Bound::Included(max))?; + table_codec.with_index_statistics_bound(table_id, index_id, |min, max, pool| { + let mut iter = self.range(Bound::Included(min), Bound::Included(max), pool)?; let mut root = None; let mut buckets = Vec::new(); let mut sketch_meta = None; @@ -1093,6 +1093,7 @@ pub trait StorageTransaction: Sized { } } } + iter.recycle(pool); match (root, sketch_meta) { (Some(root), Some(sketch_meta)) => { @@ -1117,8 +1118,8 @@ pub trait StorageTransaction: Sized { table_id: TableId, index_id: IndexId, ) -> Result<(), DatabaseError> { - table_codec.with_index_statistics_bound(table_id, index_id, |min, max| { - self.remove_range(Bound::Included(min), Bound::Included(max)) + table_codec.with_index_statistics_bound(table_id, index_id, |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) })?; Ok(()) @@ -1131,9 +1132,9 @@ pub trait StorageTransaction: Sized { table_id: TableId, arena: &mut (impl MetaArena + ?Sized), ) -> Result, Vec)>, DatabaseError> { - table_codec.with_table_bound(table_id, |table_min, table_max| { + table_codec.with_table_bound(table_id, |table_min, table_max, pool| { let mut column_iter = - self.range(Bound::Included(table_min), Bound::Included(table_max))?; + self.range(Bound::Included(table_min), Bound::Included(table_max), pool)?; let mut columns = Vec::new(); let mut index_metas = Vec::new(); @@ -1147,6 +1148,7 @@ pub trait StorageTransaction: Sized { index_metas.push(TableCodec::decode_index_meta::(value, arena)?); } } + column_iter.recycle(pool); Ok((!columns.is_empty()).then_some((columns, index_metas))) }) @@ -1227,15 +1229,22 @@ pub trait StorageTransaction: Sized { &'txn self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, + pool: &mut BytesPool, ) -> Result, DatabaseError>; fn range_rev<'txn, 'key>( &'txn self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, + pool: &mut BytesPool, ) -> Result, DatabaseError>; - fn remove_range(&mut self, min: Bound<&[u8]>, max: Bound<&[u8]>) -> Result<(), DatabaseError> { + fn remove_range( + &mut self, + min: Bound<&[u8]>, + max: Bound<&[u8]>, + pool: &mut BytesPool, + ) -> Result<(), DatabaseError> { const DELETE_BATCH_SIZE: usize = 1024; let mut lower = owned_bound(min); @@ -1244,8 +1253,11 @@ pub trait StorageTransaction: Sized { loop { data_keys.clear(); - let mut iter = - self.range(bytes_bound_as_slice(&lower), bytes_bound_as_slice(&upper))?; + let mut iter = self.range( + bytes_bound_as_slice(&lower), + bytes_bound_as_slice(&upper), + pool, + )?; while data_keys.len() < DELETE_BATCH_SIZE { let Some((key, _)) = iter.try_next()? else { @@ -1253,7 +1265,7 @@ pub trait StorageTransaction: Sized { }; data_keys.push(key.to_vec()); } - drop(iter); + iter.recycle(pool); let Some(last_key) = data_keys.pop() else { return Ok(()); @@ -1502,13 +1514,21 @@ impl<'a, T: StorageTransaction + 'a> RangeIter<'a, T> { min: Bound<&[u8]>, max: Bound<&[u8]>, reverse: bool, + pool: &mut BytesPool, ) -> Result { Ok(if reverse { - RangeIter::Reverse(tx.range_rev(min, max)?) + RangeIter::Reverse(tx.range_rev(min, max, pool)?) } else { - RangeIter::Forward(tx.range(min, max)?) + RangeIter::Forward(tx.range(min, max, pool)?) }) } + + fn recycle(self, pool: &mut BytesPool) { + match self { + RangeIter::Forward(iter) => iter.recycle(pool), + RangeIter::Reverse(iter) => iter.recycle(pool), + } + } } impl<'a, T: StorageTransaction + 'a> InnerIter for RangeIter<'a, T> { @@ -1948,6 +1968,7 @@ fn eq_to_res_scope<'a, T: StorageTransaction + 'a>( Bound::Included(encode_min.as_slice()), Bound::Included(encode_max.as_slice()), params.reverse, + table_codec.bytes_pool(), )?; Ok(IndexResult::Scope(iter)) } @@ -2041,15 +2062,20 @@ pub struct TupleIter<'a, T: StorageTransaction + 'a> { } impl<'a, T: StorageTransaction + 'a> TupleIter<'a, T> { + pub(crate) fn recycle(self, pool: &mut BytesPool) { + self.iter.recycle(pool); + } + pub(crate) fn reset( &mut self, tx: &'a T, table_codec: &mut TableCodec, bounds: Bounds, ) -> Result<(), DatabaseError> { - self.iter = table_codec.with_tuple_bound(self.table_id, |min, max, _| { - tx.range(Bound::Included(min), Bound::Included(max)) + let iter = table_codec.with_tuple_bound(self.table_id, |min, max, pool| { + tx.range(Bound::Included(min), Bound::Included(max), pool) })?; + mem::replace(&mut self.iter, iter).recycle(table_codec.bytes_pool()); self.mapped = tx.mapped_scope(); self.bounds = IterBounds::new(bounds.0.unwrap_or(0), bounds.1); Ok(()) @@ -2231,6 +2257,14 @@ impl<'a, T: StorageTransaction> IndexIter<'a, T> { self.rewind(bounds); } + pub(crate) fn recycle(self, pool: &mut BytesPool) { + pool.recycle_buffer(self.encode_min_buffer); + pool.recycle_buffer(self.encode_max_buffer); + if let IndexIterState::Range(iter) = self.state { + iter.recycle(pool); + } + } + pub(crate) fn rewind(&mut self, bounds: Bounds) { self.state = IndexIterState::Init; self.ranges.next = 0; @@ -2253,6 +2287,12 @@ impl Iter for IndexIter<'_, T> { loop { match &mut self.state { IndexIterState::Init => { + // Encode bounds into buffers that earlier iterators handed back. + for buffer in [&mut self.encode_min_buffer, &mut self.encode_max_buffer] { + if buffer.capacity() == 0 { + *buffer = table_codec.bytes_pool().buffer(); + } + } let Some(binary) = self.ranges.next() else { self.state = IndexIterState::Over; continue; @@ -2280,17 +2320,18 @@ impl Iter for IndexIter<'_, T> { let tx = self.params.tx; let reverse = self.params.reverse; - let open_iter = move |bound_min: &[u8], bound_max: &[u8]| { - RangeIter::open( - tx, - fill_default_bound(encode_min, bound_min), - fill_default_bound(encode_max, bound_max), - reverse, - ) - }; + let open_iter = + move |bound_min: &[u8], bound_max: &[u8], pool: &mut BytesPool| { + RangeIter::open( + tx, + fill_default_bound(encode_min, bound_min), + fill_default_bound(encode_max, bound_max), + reverse, + pool, + ) + }; let iter = if matches!(index_meta.ty, IndexType::PrimaryKey { .. }) { - table_codec - .with_tuple_bound(table_id, |min, max, _| open_iter(min, max))? + table_codec.with_tuple_bound(table_id, open_iter)? } else { table_codec.with_index_bound(table_id, index_meta.id, open_iter)? }; @@ -2342,7 +2383,11 @@ impl Iter for IndexIter<'_, T> { return Ok(true); } - self.state = IndexIterState::Init; + if let IndexIterState::Range(iter) = + mem::replace(&mut self.state, IndexIterState::Init) + { + iter.recycle(table_codec.bytes_pool()); + } } IndexIterState::Over => return Ok(false), } @@ -2352,6 +2397,14 @@ impl Iter for IndexIter<'_, T> { pub trait InnerIter { fn try_next(&mut self) -> Result>, DatabaseError>; + + #[doc(hidden)] + fn recycle(self, pool: &mut BytesPool) + where + Self: Sized, + { + let _ = pool; + } } pub trait Iter { @@ -2363,7 +2416,7 @@ pub trait Iter { } pub struct TableIter<'a, T: StorageTransaction + 'a> { - iter: T::IterType<'a>, + pub(crate) iter: T::IterType<'a>, } impl TableIter<'_, T> { @@ -2383,7 +2436,7 @@ impl TableIter<'_, T> { } pub struct ViewIter<'a, T: StorageTransaction + 'a> { - iter: T::IterType<'a>, + pub(crate) iter: T::IterType<'a>, } impl ViewIter<'_, T> { @@ -2439,6 +2492,7 @@ pub(crate) fn check_range_rev_matches_range( b"z", b"zz", ]; + let mut pool = BytesPool::default(); let mut bounds = vec![(None, Bound::Unbounded)]; for (rank, probe) in probes.iter().enumerate() { bounds.push((Some(rank), Bound::Included(*probe))); @@ -2463,12 +2517,12 @@ pub(crate) fn check_range_rev_matches_range( .cloned() .collect(); assert_eq!( - keys(tx.range(*min, *max)?)?, + keys(tx.range(*min, *max, &mut pool)?)?, expected, "min={min:?} max={max:?}" ); expected.reverse(); - let actual = keys(tx.range_rev(*min, *max)?)?; + let actual = keys(tx.range_rev(*min, *max, &mut pool)?)?; assert_eq!(actual, expected, "min={min:?} max={max:?}"); } } @@ -2515,13 +2569,14 @@ pub(crate) fn check_remove_range(tx: &mut T) -> Result<() (Bound::Unbounded, Bound::Excluded(&k1)), (Bound::Included(&k5000), Bound::Unbounded), ]; + let mut pool = BytesPool::default(); for (min, max) in cases { for key in &data { tx.set(key, &[7; 64])?; } - tx.remove_range(min, max)?; + tx.remove_range(min, max, &mut pool)?; - let mut iter = tx.range(Bound::Unbounded, Bound::Unbounded)?; + let mut iter = tx.range(Bound::Unbounded, Bound::Unbounded, &mut pool)?; let mut remaining = Vec::new(); while let Some((key, _)) = iter.try_next()? { remaining.push(key.to_vec()); @@ -2830,8 +2885,8 @@ mod test { tuples[2] ); - let mut iter = table_codec.with_tuple_bound(0, |min, max, _| { - transaction.range(Bound::Included(min), Bound::Included(max)) + let mut iter = table_codec.with_tuple_bound(0, |min, max, pool| { + transaction.range(Bound::Included(min), Bound::Included(max), pool) })?; let (_, value) = iter.try_next()?.unwrap(); @@ -2869,8 +2924,8 @@ mod test { tuples[2] ); - let mut iter = table_codec.with_tuple_bound(0, |min, max, _| { - transaction.range(Bound::Included(min), Bound::Included(max)) + let mut iter = table_codec.with_tuple_bound(0, |min, max, pool| { + transaction.range(Bound::Included(min), Bound::Included(max), pool) })?; let (_, value) = iter.try_next()?.unwrap(); @@ -2968,8 +3023,8 @@ mod test { let mut plan_arena = PlanArena::new(&table_arena); fn_assert(&mut transaction, &reloaded_names, &reloaded_table_cache)?; { - let mut iter = table_codec.with_index_meta_bound(0, |min, max| { - transaction.range(Bound::Included(min), Bound::Included(max)) + let mut iter = table_codec.with_index_meta_bound(0, |min, max, pool| { + transaction.range(Bound::Included(min), Bound::Included(max), pool) })?; let (_, value) = iter.try_next()?.unwrap(); @@ -3000,8 +3055,8 @@ mod test { assert_eq!(i2_meta.name, "i2".to_string()); assert_eq!(i2_meta.ty, IndexType::Composite); - let mut iter = table_codec.with_index_meta_bound(0, |min, max| { - transaction.range(Bound::Included(min), Bound::Included(max)) + let mut iter = table_codec.with_index_meta_bound(0, |min, max, pool| { + transaction.range(Bound::Included(min), Bound::Included(max), pool) })?; let (_, value) = iter.try_next()?.unwrap(); @@ -3153,8 +3208,8 @@ mod test { tuples[1] ); - let mut iter = table_codec.with_index_bound(0, 1, |min, max| { - transaction.range(Bound::Included(min), Bound::Included(max)) + let mut iter = table_codec.with_index_bound(0, 1, |min, max, pool| { + transaction.range(Bound::Included(min), Bound::Included(max), pool) })?; let (_, value) = iter.try_next()?.unwrap(); @@ -3193,8 +3248,8 @@ mod test { tuples[1] ); - let mut iter = table_codec.with_index_bound(0, 1, |min, max| { - transaction.range(Bound::Included(min), Bound::Included(max)) + let mut iter = table_codec.with_index_bound(0, 1, |min, max, pool| { + transaction.range(Bound::Included(min), Bound::Included(max), pool) })?; let (_, value) = iter.try_next()?.unwrap(); diff --git a/src/storage/rocksdb.rs b/src/storage/rocksdb.rs index ba2f2a6a..6cda8375 100644 --- a/src/storage/rocksdb.rs +++ b/src/storage/rocksdb.rs @@ -16,6 +16,7 @@ use crate::errors::DatabaseError; use crate::storage::{ CheckpointableStorage, InnerIter, Storage, StorageTransaction, TransactionIsolationLevel, }; +use crate::types::serialize::BytesPool; use rocksdb::{ checkpoint::Checkpoint, statistics::{StatsLevel, Ticker}, @@ -608,6 +609,7 @@ macro_rules! impl_transaction { &'a self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, + _: &mut BytesPool, ) -> Result, DatabaseError> { let mut read_opts = build_read_options(&self.snapshot); set_prefix_same_as_start(&mut read_opts, min, max); @@ -636,6 +638,7 @@ macro_rules! impl_transaction { &'a self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, + _: &mut BytesPool, ) -> Result, DatabaseError> { let mut read_opts = build_read_options(&self.snapshot); set_prefix_same_as_start(&mut read_opts, min, max); @@ -801,6 +804,7 @@ mod test { PrimaryKeyIndexImpl, Storage, StorageTransaction, }; use crate::types::index::IndexType; + use crate::types::serialize::BytesPool; use crate::types::tuple::Tuple; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -915,14 +919,17 @@ mod test { transaction.set(b"b\0", b"3")?; transaction.set(b"c", b"4")?; - let mut inclusive_iter = transaction.range(Bound::Included(b"a"), Bound::Included(b"b"))?; + let mut pool = BytesPool::default(); + let mut inclusive_iter = + transaction.range(Bound::Included(b"a"), Bound::Included(b"b"), &mut pool)?; let mut inclusive_keys = Vec::new(); while let Some((key, _)) = inclusive_iter.try_next()? { inclusive_keys.push(key.to_vec()); } assert_eq!(inclusive_keys, vec![b"a".to_vec(), b"b".to_vec()]); - let mut exclusive_iter = transaction.range(Bound::Included(b"a"), Bound::Excluded(b"b"))?; + let mut exclusive_iter = + transaction.range(Bound::Included(b"a"), Bound::Excluded(b"b"), &mut pool)?; let mut exclusive_keys = Vec::new(); while let Some((key, _)) = exclusive_iter.try_next()? { exclusive_keys.push(key.to_vec()); diff --git a/src/storage/table_codec.rs b/src/storage/table_codec.rs index d4a46d32..b448a03a 100644 --- a/src/storage/table_codec.rs +++ b/src/storage/table_codec.rs @@ -358,18 +358,18 @@ impl TableCodec { pub fn with_index_meta_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); - self.with_table_buffers(table_id, |lower, prefix, upper| { - Self::write_key_prefix(lower, CodecType::IndexMeta, prefix); - lower.push(BOUND_MIN_TAG); + let prefix = table_id.to_be_bytes(); + let [lower, upper] = &mut self.buffers; + Self::write_key_prefix(lower, CodecType::IndexMeta, prefix); + lower.push(BOUND_MIN_TAG); - Self::write_key_prefix(upper, CodecType::IndexMeta, prefix); - upper.push(BOUND_MAX_TAG); + Self::write_key_prefix(upper, CodecType::IndexMeta, prefix); + upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice()) - }) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) } /// Range bounds covering a single secondary index. @@ -377,40 +377,40 @@ impl TableCodec { &mut self, table_id: TableId, index_id: IndexId, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); - self.with_table_buffers(table_id, |lower, prefix, upper| { - Self::write_key_prefix(lower, CodecType::Index, prefix); - lower.push(BOUND_MIN_TAG); - lower.extend_from_slice(&index_id.to_le_bytes()); - lower.push(BOUND_MIN_TAG); + let prefix = table_id.to_be_bytes(); + let [lower, upper] = &mut self.buffers; + Self::write_key_prefix(lower, CodecType::Index, prefix); + lower.push(BOUND_MIN_TAG); + lower.extend_from_slice(&index_id.to_le_bytes()); + lower.push(BOUND_MIN_TAG); - Self::write_key_prefix(upper, CodecType::Index, prefix); - upper.push(BOUND_MIN_TAG); - upper.extend_from_slice(&index_id.to_le_bytes()); - upper.push(BOUND_MAX_TAG); + Self::write_key_prefix(upper, CodecType::Index, prefix); + upper.push(BOUND_MIN_TAG); + upper.extend_from_slice(&index_id.to_le_bytes()); + upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice()) - }) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) } /// Range bounds covering all secondary indexes for a table. pub fn with_all_index_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); - self.with_table_buffers(table_id, |lower, prefix, upper| { - Self::write_key_prefix(lower, CodecType::Index, prefix); - lower.push(BOUND_MIN_TAG); + let prefix = table_id.to_be_bytes(); + let [lower, upper] = &mut self.buffers; + Self::write_key_prefix(lower, CodecType::Index, prefix); + lower.push(BOUND_MIN_TAG); - Self::write_key_prefix(upper, CodecType::Index, prefix); - upper.push(BOUND_MAX_TAG); + Self::write_key_prefix(upper, CodecType::Index, prefix); + upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice()) - }) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) } /// Non-unique index key: @@ -496,54 +496,54 @@ impl TableCodec { pub fn with_columns_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); - self.with_table_buffers(table_id, |lower, prefix, upper| { - Self::write_key_prefix(lower, CodecType::Column, prefix); - lower.push(BOUND_MIN_TAG); + let prefix = table_id.to_be_bytes(); + let [lower, upper] = &mut self.buffers; + Self::write_key_prefix(lower, CodecType::Column, prefix); + lower.push(BOUND_MIN_TAG); - Self::write_key_prefix(upper, CodecType::Column, prefix); - upper.push(BOUND_MAX_TAG); + Self::write_key_prefix(upper, CodecType::Column, prefix); + upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice()) - }) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) } /// Range bounds spanning a table's `Column` and `IndexMeta` metadata. pub fn with_table_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); - self.with_table_buffers(table_id, |lower, prefix, upper| { - Self::write_key_prefix(lower, CodecType::Column, prefix); - lower.push(BOUND_MIN_TAG); + let prefix = table_id.to_be_bytes(); + let [lower, upper] = &mut self.buffers; + Self::write_key_prefix(lower, CodecType::Column, prefix); + lower.push(BOUND_MIN_TAG); - Self::write_key_prefix(upper, CodecType::IndexMeta, prefix); - upper.push(BOUND_MAX_TAG); + Self::write_key_prefix(upper, CodecType::IndexMeta, prefix); + upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice()) - }) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) } /// Range bounds covering all statistics keys for a table. pub fn with_statistics_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); - self.with_table_buffers(table_id, |lower, prefix, upper| { - Self::write_key_prefix(lower, CodecType::Statistics, prefix); - lower.push(BOUND_MIN_TAG); + let prefix = table_id.to_be_bytes(); + let [lower, upper] = &mut self.buffers; + Self::write_key_prefix(lower, CodecType::Statistics, prefix); + lower.push(BOUND_MIN_TAG); - Self::write_key_prefix(upper, CodecType::Statistics, prefix); - upper.push(BOUND_MAX_TAG); + Self::write_key_prefix(upper, CodecType::Statistics, prefix); + upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice()) - }) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) } /// Range bounds covering all statistics keys for one index. @@ -551,21 +551,21 @@ impl TableCodec { &mut self, table_id: TableId, index_id: IndexId, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); - self.with_table_buffers(table_id, |lower, prefix, upper| { - Self::write_key_prefix(lower, CodecType::Statistics, prefix); - lower.push(BOUND_MIN_TAG); - lower.extend_from_slice(&index_id.to_le_bytes()); - lower.push(INDEX_STATISTICS_TAG); + let prefix = table_id.to_be_bytes(); + let [lower, upper] = &mut self.buffers; + Self::write_key_prefix(lower, CodecType::Statistics, prefix); + lower.push(BOUND_MIN_TAG); + lower.extend_from_slice(&index_id.to_le_bytes()); + lower.push(INDEX_STATISTICS_TAG); - upper.clear(); - upper.extend_from_slice(lower); - upper.push(BOUND_MAX_TAG); + upper.clear(); + upper.extend_from_slice(lower); + upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice()) - }) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) } fn with_statistics( @@ -774,14 +774,14 @@ impl TableCodec { /// Range bounds covering all view definitions. pub fn with_view_bound( &mut self, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); - let (lower, upper) = self.slots_mut(); + let [lower, upper] = &mut self.buffers; Self::write_global_bound_prefix(lower, VIEW_BYTES.as_slice(), BOUND_MIN_TAG); Self::write_global_bound_prefix(upper, VIEW_BYTES.as_slice(), BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice()) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) } /// Key: `Root{BOUND_MIN_TAG}{TableName}`, for both tables and views. @@ -807,14 +807,14 @@ impl TableCodec { /// Range bounds covering all root table metadata. pub fn with_root_table_bound( &mut self, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); - let (lower, upper) = self.slots_mut(); + let [lower, upper] = &mut self.buffers; Self::write_global_bound_prefix(lower, ROOT_BYTES.as_slice(), BOUND_MIN_TAG); Self::write_global_bound_prefix(upper, ROOT_BYTES.as_slice(), BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice()) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) } /// Key: `NextTableId`, the id the next table or view gets. @@ -1334,12 +1334,12 @@ mod tests { |key, value| Ok((key.to_vec(), value.to_vec())), )?; assert_ne!(column_key, index_key); - table_codec.with_index_statistics_bound(0, 0, |min, max| { + table_codec.with_index_statistics_bound(0, 0, |min, max, _| { assert!(index_key.as_slice() >= min && index_key.as_slice() <= max); assert!(column_key.as_slice() < min || column_key.as_slice() > max); Ok(()) })?; - table_codec.with_statistics_bound(0, |min, max| { + table_codec.with_statistics_bound(0, |min, max, _| { assert!(index_key.as_slice() >= min && index_key.as_slice() <= max); assert!(column_key.as_slice() >= min && column_key.as_slice() <= max); Ok(()) @@ -1398,7 +1398,7 @@ mod tests { assert!(bucket0_key < bucket1_key); let (bucket0_min, bucket0_max) = - table_codec.with_index_statistics_bound(0, 0, |min, max| { + table_codec.with_index_statistics_bound(0, 0, |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) })?; assert!(bucket0_key.as_slice() >= bucket0_min.as_slice()); @@ -1675,7 +1675,7 @@ mod tests { set.insert(op(0, 2)); let (min, max) = table_codec - .with_columns_bound(1, |min, max| { + .with_columns_bound(1, |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) .unwrap(); @@ -1733,7 +1733,7 @@ mod tests { set.insert(op(2, 2)); let (min, max) = table_codec - .with_index_meta_bound(1, |min, max| { + .with_index_meta_bound(1, |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) .unwrap(); @@ -1795,7 +1795,7 @@ mod tests { set.insert(op(DataValue::Int32(2), 2, table_catalog.id())); let (min, max) = table_codec - .with_index_bound(table_catalog.id(), 1, |min, max| { + .with_index_bound(table_catalog.id(), 1, |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) .unwrap(); @@ -1848,7 +1848,7 @@ mod tests { set.insert(op(DataValue::Int32(2), 0, 2)); let (min, max) = table_codec - .with_all_index_bound(1, |min, max| { + .with_all_index_bound(1, |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) .unwrap(); @@ -1939,7 +1939,9 @@ mod tests { set.insert(op("T2")); let (min, max) = table_codec - .with_root_table_bound(|min, max| Ok::<_, DatabaseError>((min.to_vec(), max.to_vec()))) + .with_root_table_bound(|min, max, _| { + Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) + }) .unwrap(); let vec = set @@ -1977,7 +1979,7 @@ mod tests { set.insert(op(2)); let (min, max) = table_codec - .with_view_bound(|min, max| Ok::<_, DatabaseError>((min.to_vec(), max.to_vec()))) + .with_view_bound(|min, max, _| Ok::<_, DatabaseError>((min.to_vec(), max.to_vec()))) .unwrap(); let vec = set diff --git a/src/types/mapped.rs b/src/types/mapped.rs index b2782dc4..6fa5f276 100644 --- a/src/types/mapped.rs +++ b/src/types/mapped.rs @@ -261,6 +261,13 @@ impl Clone for MaybeMapped { fn clone(&self) -> Self { MaybeMapped(Repr::Owned((**self).to_owned())) } + + fn clone_from(&mut self, source: &Self) { + match &mut self.0 { + Repr::Owned(owned) => (**source).clone_into(owned), + Repr::Mapped(_) => *self = source.clone(), + } + } } impl Default for MaybeMapped diff --git a/src/types/serialize.rs b/src/types/serialize.rs index 20b339a4..4ff92cc2 100644 --- a/src/types/serialize.rs +++ b/src/types/serialize.rs @@ -237,8 +237,9 @@ impl TupleValueSerializable for TupleValueSerializableImpl { } } +#[doc(hidden)] #[derive(Debug, Default)] -pub(crate) struct BytesPool(Vec>); +pub struct BytesPool(Vec>); impl BytesPool { const CAPACITY: usize = 64; @@ -254,6 +255,20 @@ impl BytesPool { } } + pub(crate) fn buffer(&mut self) -> Vec { + self.0.pop().unwrap_or_default() + } + + pub(crate) fn recycle_buffer(&mut self, mut buffer: Vec) { + if self.0.len() < Self::CAPACITY + && buffer.capacity() > 0 + && buffer.capacity() <= Self::MAX_BUFFER_LEN + { + buffer.clear(); + self.0.push(buffer); + } + } + /// # Safety /// /// With a `scope`, `bytes` must be a read of the transaction holding it. @@ -308,12 +323,8 @@ impl BytesPool { DataValue::Binary { value, .. } => value.into_owned_raw(), _ => continue, }; - let Some(mut buffer) = buffer else { - continue; - }; - if self.0.len() < Self::CAPACITY && buffer.capacity() <= Self::MAX_BUFFER_LEN { - buffer.clear(); - self.0.push(buffer); + if let Some(buffer) = buffer { + self.recycle_buffer(buffer); } } } diff --git a/src/types/value.rs b/src/types/value.rs index e0cf810a..bc7eef4d 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -132,7 +132,6 @@ pub enum Utf8Type { Fixed(u32), } -#[derive(Clone)] pub enum DataValue { /// A typed placeholder used only while preparing a plan. Parameter { @@ -175,6 +174,80 @@ pub enum DataValue { }, } +impl Clone for DataValue { + fn clone(&self) -> Self { + match self { + DataValue::Parameter { id, ty } => DataValue::Parameter { + id: *id, + ty: ty.clone(), + }, + DataValue::InitValue { id, ty } => DataValue::InitValue { + id: *id, + ty: ty.clone(), + }, + DataValue::Null => DataValue::Null, + DataValue::Boolean(v) => DataValue::Boolean(*v), + DataValue::Float32(v) => DataValue::Float32(*v), + DataValue::Float64(v) => DataValue::Float64(*v), + DataValue::Int8(v) => DataValue::Int8(*v), + DataValue::Int16(v) => DataValue::Int16(*v), + DataValue::Int32(v) => DataValue::Int32(*v), + DataValue::Int64(v) => DataValue::Int64(*v), + DataValue::UInt8(v) => DataValue::UInt8(*v), + DataValue::UInt16(v) => DataValue::UInt16(*v), + DataValue::UInt32(v) => DataValue::UInt32(*v), + DataValue::UInt64(v) => DataValue::UInt64(*v), + DataValue::Utf8 { value, ty, unit } => DataValue::Utf8 { + value: value.clone(), + ty: ty.clone(), + unit: *unit, + }, + DataValue::Date32(v) => DataValue::Date32(*v), + DataValue::Date64(v) => DataValue::Date64(*v), + DataValue::Time32(v, precision) => DataValue::Time32(*v, *precision), + DataValue::Time64(v, precision, zone) => DataValue::Time64(*v, *precision, *zone), + #[cfg(feature = "decimal")] + DataValue::Decimal(v) => DataValue::Decimal(*v), + DataValue::Tuple(values) => DataValue::Tuple(values.clone()), + DataValue::Binary { value, len } => DataValue::Binary { + value: value.clone(), + len: *len, + }, + } + } + + fn clone_from(&mut self, source: &Self) { + match (self, source) { + ( + DataValue::Utf8 { value, ty, unit }, + DataValue::Utf8 { + value: source_value, + ty: source_ty, + unit: source_unit, + }, + ) => { + value.clone_from(source_value); + ty.clone_from(source_ty); + *unit = *source_unit; + } + ( + DataValue::Binary { value, len }, + DataValue::Binary { + value: source_value, + len: source_len, + }, + ) => { + value.clone_from(source_value); + *len = *source_len; + } + (DataValue::Tuple(values), DataValue::Tuple(source_values)) => { + values.clone_from(source_values) + } + (target, source) => *target = source.clone(), + } + } +} + /// A read-only view of a value, including an unmaterialized tuple slice. #[derive(Clone, Copy, Debug)] pub enum DataValueRef<'a> { From 57b48f81b0869d8c57adfbf83c3c100a6ee1cf23 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 4 Nov 2026 15:08:42 +0800 Subject: [PATCH 04/14] perf(index-scan): keep scan ranges and cover mapping on the executor - IndexScan keeps its ranges in ScanRanges: the plan's static range, or an owned range that parameters bind into with Range::bind_from, so reruns reuse its buffers. IndexIter no longer holds ranges; each call lends them with the cover mapping through IndexScanView. - The cover mapping is computed once per executor (TupleMapping::cover) instead of on every read_by_index. - Bound keys encode into ExecBuffer::bound_buffers, and StorageIter holds IndexIter unboxed (416 -> 200 bytes on LMDB). - DataValue/RangeValue/Range bind_from replace bind_parameters, writing a bound template over the previous value in place; ParamArena::bind matches the slot directly. - Range::as_slice; next_scan_tuple and next_tuple_for_test take a closure. --- src/db/program.rs | 21 +- src/execution/dql/index_scan.rs | 128 ++++++---- src/execution/dql/seq_scan.rs | 14 +- src/execution/mod.rs | 7 +- src/expression/range_detacher.rs | 214 ++++++++++++----- src/optimizer/core/statistics_meta.rs | 9 +- src/planner/arena.rs | 28 +-- src/storage/memory.rs | 58 +++-- src/storage/mod.rs | 327 ++++++++++++-------------- src/storage/rocksdb.rs | 194 +++++++++++---- src/types/serialize.rs | 4 - src/types/value.rs | 79 +++++-- 12 files changed, 674 insertions(+), 409 deletions(-) diff --git a/src/db/program.rs b/src/db/program.rs index 2c3cab10..5357d50d 100644 --- a/src/db/program.rs +++ b/src/db/program.rs @@ -225,7 +225,7 @@ impl<'p, T: StorageTransaction + 'p> ResultIter for ProgramIter<'_, 'p, T> { mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; - use crate::execution::dql::index_scan::IndexScan; + use crate::execution::dql::index_scan::{IndexScan, ScanRanges}; use crate::expression::range_detacher::{Range, RangeValue}; use crate::expression::{BinaryOperator, ScalarExpression}; use crate::planner::operator::filter::FilterOperator; @@ -421,11 +421,13 @@ mod tests { let mut exec_arena = crate::execution::ExecArena::new(context, &transaction); let mut execution_arena = crate::execution::ExecContext::new(&mut exec_buffer, &mut exec_arena); - let ranges = IndexScan::ranges( + let mut ranges = ScanRanges::Owned(Range::Dummy); + IndexScan::bind_ranges( info, info.lookup.as_ref().expect("lookup"), &mut execution_arena, &mut arena, + &mut ranges, )?; let expected_range = match &expected_index.lookup { Some(IndexLookup::Static(range)) => range, @@ -437,8 +439,7 @@ mod tests { } else { original }; - let mut ranges = ranges; - assert_eq!(ranges.next(), Some(want)); + assert_eq!(ranges.range(), want); } assert_eq!( scan.physical_option.as_ref().unwrap().plan, @@ -778,25 +779,25 @@ mod tests { let mut exec_arena = crate::execution::ExecArena::new(context, &transaction); let mut execution_arena = crate::execution::ExecContext::new(&mut exec_buffer, &mut exec_arena); - let ranges = IndexScan::ranges( + let mut ranges = ScanRanges::Owned(Range::Dummy); + IndexScan::bind_ranges( info, info.lookup.as_ref().expect("lookup"), &mut execution_arena, &mut plan.arena, + &mut ranges, )?; - let mut ranges = ranges; assert_eq!( - ranges.next(), - Some(&Range::Scope { + ranges.range(), + &Range::Scope { min: Bound::Included( DataValue::Tuple(vec![DataValue::Int32(2), DataValue::Int32(2),]).into() ), max: Bound::Excluded( DataValue::Tuple(vec![DataValue::Int32(2), DataValue::Int32(5),]).into() ), - }) + } ); - drop(ranges); // The computed bound follows each execution's parameters, and a NULL one matches nothing. let mut tx = db.new_transaction(false)?; diff --git a/src/execution/dql/index_scan.rs b/src/execution/dql/index_scan.rs index 308d8563..474a0cdb 100644 --- a/src/execution/dql/index_scan.rs +++ b/src/execution/dql/index_scan.rs @@ -17,19 +17,34 @@ use crate::execution::dql::seq_scan::next_scan_tuple; use crate::execution::{ ExecBuilder, ExecContext, ExecNode, ExecutorNode, IterSlot, ReadExecutor, StorageIter, }; -use crate::expression::range_detacher::{IndexRangeColumn, RangeDetacher}; +use crate::expression::range_detacher::{IndexRangeColumn, Range, RangeDetacher, RangeValue}; use crate::planner::operator::table_scan::TableScanOperator; use crate::planner::operator::SortOption; use crate::planner::{ExecMetaArena, MetaArena}; -use crate::storage::{IndexRanges, StorageTransaction}; -use crate::types::index::{IndexInfo, IndexLookup}; -use std::borrow::Cow; +use crate::storage::{IndexScanView, StorageTransaction, TupleMapping}; +use crate::types::index::{IndexInfo, IndexLookup, RuntimeIndexProbe}; pub(crate) struct IndexScan<'a> { op: &'a TableScanOperator, info: &'a IndexInfo, lookup: &'a IndexLookup, iter: IterSlot, + ranges: ScanRanges<'a>, + cover_mapping: Option, +} + +pub(crate) enum ScanRanges<'a> { + Static(&'a Range), + Owned(Range), +} + +impl ScanRanges<'_> { + pub(crate) fn range(&self) -> &Range { + match self { + ScanRanges::Static(range) => range, + ScanRanges::Owned(range) => range, + } + } } impl<'a> IndexScan<'a> { @@ -38,33 +53,62 @@ impl<'a> IndexScan<'a> { info: &'a IndexInfo, lookup: &'a IndexLookup, iter: IterSlot, + plan_arena: &(dyn MetaArena + 'a), ) -> Self { + let cover_mapping = info + .covered + .then(|| { + TupleMapping::cover( + plan_arena.index(info.meta), + &op.deserializers, + info.cover_mapping.as_deref(), + ) + }) + .flatten(); Self { op, info, lookup, iter, + ranges: ScanRanges::Owned(Range::Dummy), + cover_mapping, } } - pub(crate) fn ranges( + /// Binds the ranges of this execution into `ranges`. + pub(crate) fn bind_ranges( info: &'a IndexInfo, lookup: &'a IndexLookup, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut (dyn MetaArena + 'a), - ) -> Result, DatabaseError> { - let mut range = match lookup { - IndexLookup::Static(range) => Cow::Borrowed(range), - IndexLookup::Probe => Cow::Owned(arena.pop_runtime_probe().into()), - }; - - if range.has_parameter() { - let plan_arena = &*plan_arena; - range - .to_mut() - .bind_parameters(&|id| plan_arena.bound_param(id), &|id| { - plan_arena.init_value(id) - })?; + ranges: &mut ScanRanges<'a>, + ) -> Result<(), DatabaseError> { + match lookup { + IndexLookup::Static(template) if !template.has_parameter() => { + *ranges = ScanRanges::Static(template) + } + IndexLookup::Static(template) => { + let plan_arena = &*plan_arena; + let bind = |range: &mut Range| { + range.bind_from(template, &|id| plan_arena.bound_param(id), &|id| { + plan_arena.init_value(id) + }) + }; + match ranges { + ScanRanges::Owned(range) => bind(range)?, + ScanRanges::Static(_) => { + let mut range = Range::Dummy; + bind(&mut range)?; + *ranges = ScanRanges::Owned(range); + } + } + } + IndexLookup::Probe => match (&mut *ranges, arena.pop_runtime_probe()) { + (ScanRanges::Owned(Range::Eq(value)), RuntimeIndexProbe::Eq(probe)) => { + *value = RangeValue::Value(probe) + } + (ranges, probe) => *ranges = ScanRanges::Owned(probe.into()), + }, } if let ( Some(predicate), @@ -75,15 +119,15 @@ impl<'a> IndexScan<'a> { { if let Some(specialized) = RangeDetacher::::specialize_range( info.meta, - &range, + ranges.range(), predicate, *ignore_prefix_len, plan_arena, )? { - range = Cow::Owned(specialized); + *ranges = ScanRanges::Owned(specialized); } } - Ok(IndexRanges::from(range).reversed(info.is_reverse())) + Ok(()) } } @@ -93,10 +137,10 @@ impl<'a> ReadExecutor<'a> for IndexScan<'a> { fn into_executor( (op, info, lookup): Self::Input, arena: &mut B, - _plan_arena: &(dyn MetaArena + 'a), + plan_arena: &(dyn MetaArena + 'a), ) -> ExecNode<'a> { let iter = arena.alloc_iter(); - ExecNode::IndexScan(Box::new(IndexScan::new(op, info, lookup, iter))) + ExecNode::IndexScan(Box::new(IndexScan::new(op, info, lookup, iter, plan_arena))) } } @@ -110,20 +154,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexScan<'a> { if arena.iter_mut(slot).is_empty() { return Ok(()); } - match self.lookup { - IndexLookup::Static(_) => arena.iter_mut(slot).as_index().rewind(self.op.limit), - IndexLookup::Probe if self.info.residual_predicate.is_none() => { - let probe = arena.pop_runtime_probe(); - arena - .iter_mut(slot) - .as_index() - .reset_probe(probe, self.op.limit) - } - IndexLookup::Probe => { - let ranges = Self::ranges(self.info, self.lookup, arena, plan_arena)?; - arena.iter_mut(slot).as_index().reset(ranges, self.op.limit); - } + if let IndexLookup::Probe = self.lookup { + Self::bind_ranges(self.info, self.lookup, arena, plan_arena, &mut self.ranges)?; } + arena.iter_mut(slot).as_index().rewind(self.op.limit); Ok(()) } @@ -133,7 +167,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexScan<'a> { plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if arena.iter_mut(self.iter).is_empty() { - let ranges = Self::ranges(self.info, self.lookup, arena, plan_arena)?; + Self::bind_ranges(self.info, self.lookup, arena, plan_arena, &mut self.ranges)?; let TableScanOperator { table_id, deserializers, @@ -147,19 +181,29 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexScan<'a> { *table_id, *limit, self.info.meta, - ranges, + self.info.is_reverse(), *with_pk, deserializers, self.info.covered, - self.info.cover_mapping.as_deref(), )?; - state.iters[self.iter] = StorageIter::Index(Box::new(iter)); + state.iters[self.iter] = StorageIter::Index(iter); } + let view = IndexScanView { + range: self.ranges.range(), + cover_mapping: self.cover_mapping.as_ref(), + }; let state = arena.local_state(plan_arena); let iter = state.iters[self.iter].as_index(); + let bound_buffers = state.bound_buffers; let output = state.result.output_mut(); - if next_scan_tuple(self.op, iter, state.table_codec, output, plan_arena)? { + if next_scan_tuple( + self.op, + |table_codec, tuple| iter.next_tuple_into(view, bound_buffers, table_codec, tuple), + state.table_codec, + output, + plan_arena, + )? { arena.resume(); } else { arena.finish(); @@ -241,7 +285,7 @@ mod tests { let mut arena = crate::execution::ExecContext::new(&mut exec_buffer, &mut exec_arena); let mut meta = ExecMetaArena::new(meta); let slot = arena.alloc_iter(); - let mut scan = IndexScan::new(&op, &info, &IndexLookup::Probe, slot); + let mut scan = IndexScan::new(&op, &info, &IndexLookup::Probe, slot, &meta); arena.push_runtime_probe(RuntimeIndexProbe::Scope { min: Bound::Included(DataValue::Int32(1)), max: Bound::Included(DataValue::Int32(4)), diff --git a/src/execution/dql/seq_scan.rs b/src/execution/dql/seq_scan.rs index 98957f27..6e06c206 100644 --- a/src/execution/dql/seq_scan.rs +++ b/src/execution/dql/seq_scan.rs @@ -77,7 +77,13 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SeqScan<'a> { } let iter = state.iters[self.iter].as_tuple(); let output = state.result.output_mut(); - if next_scan_tuple(self.op, iter, state.table_codec, output, plan_arena)? { + if next_scan_tuple( + self.op, + |table_codec, tuple| iter.next_tuple_into(table_codec, tuple), + state.table_codec, + output, + plan_arena, + )? { arena.resume(); } else { arena.finish(); @@ -87,14 +93,14 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SeqScan<'a> { } #[inline] -pub(crate) fn next_scan_tuple( +pub(crate) fn next_scan_tuple( op: &TableScanOperator, - iter: &mut I, + mut next: impl FnMut(&mut TableCodec, &mut Tuple) -> Result, table_codec: &mut TableCodec, tuple: &mut Tuple, arena: &A, ) -> Result { - while iter.next_tuple_into(table_codec, tuple)? { + while next(table_codec, tuple)? { let holds = match op.predicate { Some(predicate) => arena .expression(predicate) diff --git a/src/execution/mod.rs b/src/execution/mod.rs index 9eaf883a..ebea95dc 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -77,7 +77,7 @@ use crate::planner::operator::join::JoinCondition; use crate::planner::operator::{Operator, PhysicalOption, PlanImpl}; use crate::planner::MetaArena; use crate::planner::{ExecMetaArena, ExprRef, LogicalPlan, PlanArena, PlanKeeper}; -use crate::storage::table_codec::TableCodec; +use crate::storage::table_codec::{Bytes, TableCodec}; use crate::storage::{ IndexIter, InnerIter, StorageTransaction, TableCache, TableIter, TupleIter, ViewIter, }; @@ -583,6 +583,7 @@ pub(crate) struct ExecBuffer { table_codec: TableCodec, runtime_probe_stack: Vec, restarting: bool, + bound_buffers: [Bytes; 2], } pub(crate) struct ExecArena<'a, 't, T: StorageTransaction + 'a> { @@ -602,7 +603,7 @@ pub(crate) struct ExecContext<'c, 'a, 't, T: StorageTransaction + 'a> { pub(crate) enum StorageIter<'t, T: StorageTransaction + 't> { Empty, Tuple(TupleIter<'t, T>), - Index(Box>), + Index(IndexIter<'t, T>), Table(TableIter<'t, T>), View(ViewIter<'t, T>), } @@ -654,6 +655,7 @@ impl<'t, T: StorageTransaction + 't> StorageIter<'t, T> { pub(crate) struct ExecArenaLocalState<'b, 'a, 't, T: StorageTransaction + 'a> { transaction: *mut T, pub(crate) table_codec: &'b mut TableCodec, + pub(crate) bound_buffers: &'b mut [Bytes; 2], pub(crate) result: &'b mut ExecResult, pub(crate) plan_arena: &'b (dyn MetaArena + 'a), pub(crate) iters: &'b mut [StorageIter<'t, T>], @@ -780,6 +782,7 @@ impl<'c, 'a, 't, T: StorageTransaction + 'a> ExecContext<'c, 'a, 't, T> { ExecArenaLocalState { transaction: self.arena.transaction, table_codec: &mut self.buffer.table_codec, + bound_buffers: &mut self.buffer.bound_buffers, result: &mut self.buffer.result, plan_arena, iters: &mut self.arena.iters, diff --git a/src/expression/range_detacher.rs b/src/expression/range_detacher.rs index b838a5cd..fd357d5a 100644 --- a/src/expression/range_detacher.rs +++ b/src/expression/range_detacher.rs @@ -158,8 +158,9 @@ impl RangeValue { } } - fn bind_parameters<'p, F, G>( + fn bind_from<'p, F, G>( &mut self, + template: &RangeValue, params: &F, init_values: &G, ) -> Result @@ -167,13 +168,15 @@ impl RangeValue { F: Fn(usize) -> Option<&'p DataValue>, G: Fn(ScalarQueryRef) -> Option<&'p DataValue>, { + let bind = |value: &mut DataValue| match template { + RangeValue::Value(template) => value.bind_from(template, params, init_values), + RangeValue::Computed(computed) => computed.compute(value, params, init_values), + }; match self { - RangeValue::Value(value) => value.bind_parameters(params, init_values), - RangeValue::Computed(computed) => { - let Some(mut value) = computed.compute(params, init_values)? else { - return Ok(true); - }; - let has_null = value.bind_parameters(params, init_values)?; + RangeValue::Value(value) => bind(value), + RangeValue::Computed(_) => { + let mut value = DataValue::Null; + let has_null = bind(&mut value)?; *self = RangeValue::Value(value); Ok(has_null) } @@ -189,20 +192,22 @@ impl RangeValue { } impl ComputedValue { - /// Computes the value, moving the operands out; `None` when the result is NULL. + /// Computes the value into `target`. Returns whether the result or a bound prefix value is + /// NULL. fn compute<'p, F, G>( - &mut self, + &self, + target: &mut DataValue, params: &F, init_values: &G, - ) -> Result, DatabaseError> + ) -> Result where F: Fn(usize) -> Option<&'p DataValue>, G: Fn(ScalarQueryRef) -> Option<&'p DataValue>, { - let mut left = mem::replace(&mut self.left, DataValue::Null); - let mut right = mem::replace(&mut self.right, DataValue::Null); - left.bind_parameters(params, init_values)?; - right.bind_parameters(params, init_values)?; + let mut left = DataValue::Null; + let mut right = DataValue::Null; + left.bind_from(&self.left, params, init_values)?; + right.bind_from(&self.right, params, init_values)?; let value = binary_create(Cow::Borrowed(&self.eval_ty), self.op)? .binary_eval( Cow::Owned(left.cast(&self.eval_ty)?), @@ -210,15 +215,26 @@ impl ComputedValue { )? .cast(&self.ty)?; if value.is_null() { - return Ok(None); + return Ok(true); } - Ok(Some(match self.prefix.take() { - Some(mut values) => { - values.push(value); - DataValue::Tuple(values) + let Some(prefix) = &self.prefix else { + *target = value; + return Ok(false); + }; + let fill = |values: &mut Vec| { + let has_null = DataValue::bind_values_from(values, prefix, params, init_values)?; + values.push(value); + Ok(has_null) + }; + match target { + DataValue::Tuple(values) => fill(values), + _ => { + let mut values = Vec::with_capacity(prefix.len() + 1); + let has_null = fill(&mut values)?; + *target = DataValue::Tuple(values); + Ok(has_null) } - None => value, - })) + } } } @@ -403,8 +419,16 @@ fn build_tree( } impl Range { - pub(crate) fn bind_parameters<'p, F, G>( + pub(crate) fn as_slice(&self) -> &[Range] { + match self { + Range::SortedRanges(ranges) => ranges, + range => std::slice::from_ref(range), + } + } + + pub(crate) fn bind_from<'p, F, G>( &mut self, + template: &Range, params: &F, init_values: &G, ) -> Result<(), DatabaseError> @@ -412,24 +436,82 @@ impl Range { F: Fn(usize) -> Option<&'p DataValue>, G: Fn(ScalarQueryRef) -> Option<&'p DataValue>, { - let has_null = match self { + let has_null = match template { Self::Scope { min, max } => { - let mut has_null = false; - for bound in [min, max] { - if let Bound::Included(value) | Bound::Excluded(value) = bound { - has_null |= value.bind_parameters(params, init_values)?; + let bind_bound = + |target: &mut Bound, template: &Bound| match ( + target, template, + ) { + (Bound::Included(value), Bound::Included(template)) + | (Bound::Excluded(value), Bound::Excluded(template)) => { + value.bind_from(template, params, init_values) + } + (target, Bound::Unbounded) => { + *target = Bound::Unbounded; + Ok(false) + } + (target, Bound::Included(template)) => { + let mut value = RangeValue::NULL; + let has_null = value.bind_from(template, params, init_values)?; + *target = Bound::Included(value); + Ok(has_null) + } + (target, Bound::Excluded(template)) => { + let mut value = RangeValue::NULL; + let has_null = value.bind_from(template, params, init_values)?; + *target = Bound::Excluded(value); + Ok(has_null) + } + }; + match self { + Self::Scope { + min: target_min, + max: target_max, + } => bind_bound(target_min, min)? | bind_bound(target_max, max)?, + _ => { + let (mut target_min, mut target_max) = (Bound::Unbounded, Bound::Unbounded); + let has_null = + bind_bound(&mut target_min, min)? | bind_bound(&mut target_max, max)?; + *self = Self::Scope { + min: target_min, + max: target_max, + }; + has_null } } - has_null } - Self::Eq(value) => value.bind_parameters(params, init_values)?, - Self::SortedRanges(ranges) => { - for range in ranges { - range.bind_parameters(params, init_values)?; + Self::Eq(template) => match self { + Self::Eq(value) => value.bind_from(template, params, init_values)?, + _ => { + let mut value = RangeValue::NULL; + let has_null = value.bind_from(template, params, init_values)?; + *self = Self::Eq(value); + has_null + } + }, + Self::SortedRanges(templates) => { + let bind = |ranges: &mut Vec| { + ranges.truncate(templates.len()); + ranges.resize_with(templates.len(), || Self::Dummy); + for (range, template) in ranges.iter_mut().zip(templates) { + range.bind_from(template, params, init_values)?; + } + Ok::<_, DatabaseError>(()) + }; + match self { + Self::SortedRanges(ranges) => bind(ranges)?, + _ => { + let mut ranges = Vec::with_capacity(templates.len()); + bind(&mut ranges)?; + *self = Self::SortedRanges(ranges); + } } false } - Self::Dummy => false, + Self::Dummy => { + *self = Self::Dummy; + false + } }; if has_null { *self = Self::Dummy; @@ -3487,35 +3569,39 @@ mod test { _ => None, }; - let mut point = Range::Eq(DataValue::Tuple(vec![DataValue::Int32(1), init(first)]).into()); - assert!(point.has_parameter()); - point.bind_parameters(&|_| None, &get)?; + let template = Range::Eq(DataValue::Tuple(vec![DataValue::Int32(1), init(first)]).into()); + assert!(template.has_parameter()); + let mut point = Range::Dummy; + point.bind_from(&template, &|_| None, &get)?; assert!(!point.has_parameter()); assert_eq!( point, Range::Eq(DataValue::Tuple(vec![DataValue::Int32(1), DataValue::Int64(7)]).into()) ); - let mut null_point = - Range::Eq(DataValue::Tuple(vec![DataValue::Null, init(second)]).into()); - null_point.bind_parameters(&|_| None, &get)?; - assert_eq!(null_point, Range::Dummy); + let null_template = Range::Eq(DataValue::Tuple(vec![DataValue::Null, init(second)]).into()); + // Rebinding into the bound point overwrites it. + point.bind_from(&null_template, &|_| None, &get)?; + assert_eq!(point, Range::Dummy); - let mut ranges = Range::SortedRanges(vec![ + let template = Range::SortedRanges(vec![ Range::Eq(init(first).into()), Range::Scope { min: Bound::Excluded(init(second).into()), max: Bound::Excluded(DataValue::Null.into()), }, ]); - ranges.bind_parameters(&|_| None, &get)?; + let mut ranges = Range::Dummy; + ranges.bind_from(&template, &|_| None, &get)?; assert_eq!( ranges, Range::SortedRanges(vec![Range::Eq(DataValue::Int64(7).into()), Range::Dummy]) ); - let mut unset = Range::Eq(init(first).into()); - assert!(unset.bind_parameters(&|_| None, &|_| None).is_err()); + let unset = Range::Eq(init(first).into()); + assert!(Range::Dummy + .bind_from(&unset, &|_| None, &|_| None) + .is_err()); Ok(()) } @@ -3542,31 +3628,35 @@ mod test { ), Err((left.clone(), right.clone())), ); - let mut ranges = Range::SortedRanges(vec![left, right]); - let low = DataValue::Int32(8); - let high = DataValue::Int32(3); - ranges.bind_parameters( - &|id| match id { - 1 => Some(&low), - 2 => Some(&high), - _ => None, - }, - &|_| None, - )?; - assert!(!ranges.has_parameter()); - assert_eq!( - ranges, + let template = Range::SortedRanges(vec![left, right]); + let expected = |low: &DataValue, high: &DataValue| { Range::SortedRanges(vec![ Range::Scope { - min: Bound::Included(low.into()), - max: Bound::Unbounded + min: Bound::Included(low.clone().into()), + max: Bound::Unbounded, }, Range::Scope { min: Bound::Unbounded, - max: Bound::Excluded(high.into()) + max: Bound::Excluded(high.clone().into()), }, ]) - ); + }; + // The second binding writes over the first, as a rerun of a prepared statement does. + let mut ranges = Range::Dummy; + for (low, high) in [(8, 3), (1, 9)] { + let (low, high) = (DataValue::Int32(low), DataValue::Int32(high)); + ranges.bind_from( + &template, + &|id| match id { + 1 => Some(&low), + 2 => Some(&high), + _ => None, + }, + &|_| None, + )?; + assert!(!ranges.has_parameter()); + assert_eq!(ranges, expected(&low, &high)); + } Ok(()) } diff --git a/src/optimizer/core/statistics_meta.rs b/src/optimizer/core/statistics_meta.rs index bf207b6f..ee05f545 100644 --- a/src/optimizer/core/statistics_meta.rs +++ b/src/optimizer/core/statistics_meta.rs @@ -23,7 +23,6 @@ use crate::types::index::IndexId; use crate::types::value::DataValue; use crate::types::ColumnId; use kite_sql_serde_macros::ReferenceSerialization; -use std::slice; pub struct StatisticMetaLoader<'a> { cache: &'a StatisticsMetaCache, @@ -57,15 +56,9 @@ impl<'a> StatisticMetaLoader<'a> { let Some(entry) = self.load(table_id, index_id)? else { return Ok(None); }; - let ranges = if let Range::SortedRanges(ranges) = range { - ranges.as_slice() - } else { - slice::from_ref(range) - }; - entry .histogram() - .collect_count(ranges, entry.sketch(), entry.top_n(), buf) + .collect_count(range.as_slice(), entry.sketch(), entry.top_n(), buf) .map(|rows| Some((rows, entry.histogram().distinct_values_len()))) } } diff --git a/src/planner/arena.rs b/src/planner/arena.rs index 3015febd..6173c3ff 100644 --- a/src/planner/arena.rs +++ b/src/planner/arena.rs @@ -1223,27 +1223,25 @@ impl<'a> ParamArena<'a> { .iter() .find_map(|(candidate, value)| (*candidate == id).then_some(value)) .ok_or_else(|| DatabaseError::parameter_not_found(format!("${id}")))?; - let constant = || ArenaExpr::new(ScalarExpression::Constant(DataValue::Null)); if self.expressions.len() == slot { - self.expressions.push((Some(id), constant())); + self.expressions.push(( + Some(id), + ArenaExpr::new(ScalarExpression::Constant(DataValue::Null)), + )); } let (_, expr) = &mut self.expressions[slot]; - if !matches!( - expr, + match expr { ArenaExpr { - expression: ScalarExpression::Constant(_), + expression: ScalarExpression::Constant(value), param: None, + } => { + // Overwrite the previous execution's value, reusing its buffer. + value.clone_from(source); + if value.logical_type() != *ty { + *value = mem::replace(value, DataValue::Null).cast(ty)?; + } } - ) { - // An execution rewrote the slot; rebuild it. - *expr = constant(); - } - let ScalarExpression::Constant(value) = &mut expr.expression else { - unreachable!("the slot was just made a constant"); - }; - value.clone_from(source); - if value.logical_type() != *ty { - *value = mem::replace(value, DataValue::Null).cast(ty)?; + _ => *expr = ArenaExpr::new(ScalarExpression::Constant(source.clone().cast(ty)?)), } } Ok(()) diff --git a/src/storage/memory.rs b/src/storage/memory.rs index 3ca4e331..c25810bc 100644 --- a/src/storage/memory.rs +++ b/src/storage/memory.rs @@ -244,6 +244,7 @@ mod wasm_tests { use crate::expression::range_detacher::Range; use crate::planner::{PlanArena, TableArenaCell}; use crate::storage::table_codec::TableCodec; + use crate::storage::{IndexScanView, Iter}; use crate::types::tuple::Tuple; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -337,10 +338,12 @@ mod wasm_tests { (Some(1), Some(1)), )?; - let option_1 = crate::storage::next_tuple_for_test(&mut iter)?; + let option_1 = + crate::storage::next_tuple_for_test(|codec, tuple| iter.next_tuple_into(codec, tuple))?; assert_eq!(option_1.unwrap().pk, Some(DataValue::Int32(2))); - let option_2 = crate::storage::next_tuple_for_test(&mut iter)?; + let option_2 = + crate::storage::next_tuple_for_test(|codec, tuple| iter.next_tuple_into(codec, tuple))?; assert_eq!(option_2, None); Ok(()) @@ -375,6 +378,10 @@ mod wasm_tests { &plan_arena, true, ); + let range = Range::Scope { + min: Bound::Excluded(DataValue::Int32(0).into()), + max: Bound::Included(DataValue::Int32(2).into()), + }; let mut iter = transaction.read_by_index( &plan_arena, kite_sql @@ -385,18 +392,24 @@ mod wasm_tests { .id(), (Some(0), None), pk_index, - Range::Scope { - min: Bound::Excluded(DataValue::Int32(0).into()), - max: Bound::Included(DataValue::Int32(2).into()), - }, + false, true, &deserializers, false, - None, )?; let mut result = Vec::new(); - while let Some(tuple) = crate::storage::next_tuple_for_test(&mut iter)? { + while let Some(tuple) = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? { result.push(tuple.pk.unwrap()); } @@ -414,6 +427,7 @@ mod native_tests { use crate::expression::range_detacher::Range; use crate::planner::{PlanArena, TableArenaCell}; use crate::storage::table_codec::TableCodec; + use crate::storage::{IndexScanView, Iter}; use crate::types::tuple::Tuple; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -526,10 +540,12 @@ mod native_tests { (Some(1), Some(1)), )?; - let option_1 = crate::storage::next_tuple_for_test(&mut iter)?; + let option_1 = + crate::storage::next_tuple_for_test(|codec, tuple| iter.next_tuple_into(codec, tuple))?; assert_eq!(option_1.unwrap().pk, Some(DataValue::Int32(2))); - let option_2 = crate::storage::next_tuple_for_test(&mut iter)?; + let option_2 = + crate::storage::next_tuple_for_test(|codec, tuple| iter.next_tuple_into(codec, tuple))?; assert_eq!(option_2, None); Ok(()) @@ -564,6 +580,10 @@ mod native_tests { &plan_arena, true, ); + let range = Range::Scope { + min: Bound::Excluded(DataValue::Int32(0).into()), + max: Bound::Included(DataValue::Int32(2).into()), + }; let mut iter = transaction.read_by_index( &plan_arena, kite_sql @@ -574,18 +594,24 @@ mod native_tests { .id(), (Some(0), None), pk_index, - Range::Scope { - min: Bound::Excluded(DataValue::Int32(0).into()), - max: Bound::Included(DataValue::Int32(2).into()), - }, + false, true, &deserializers, false, - None, )?; let mut result = Vec::new(); - while let Some(tuple) = crate::storage::next_tuple_for_test(&mut iter)? { + while let Some(tuple) = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? { result.push(tuple.pk.unwrap()); } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 88b58c08..6f6dfd1b 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -34,7 +34,7 @@ use crate::optimizer::core::statistics_meta::IndexStatistics; use crate::planner::operator::alter_table::change_column::{DefaultChange, NotNullChange}; use crate::planner::{MetaArena, TableArenaCell}; use crate::storage::table_codec::{Bytes, IndexStatisticsCodecType, TableCodec, BOUND_MAX_TAG}; -use crate::types::index::{Index, IndexId, IndexMeta, IndexMetaRef, IndexType, RuntimeIndexProbe}; +use crate::types::index::{Index, IndexId, IndexMeta, IndexMetaRef, IndexType}; use crate::types::mapped::MappedScope; use crate::types::serialize::{BytesPool, TupleValueSerializableImpl}; use crate::types::tuple::{Tuple, TupleId, TupleImpl, TupleLike}; @@ -256,42 +256,18 @@ pub trait StorageTransaction: Sized { } #[allow(clippy::too_many_arguments)] - fn read_by_index<'a, R>( + fn read_by_index<'a>( &'a self, arena: &(dyn MetaArena + 'a), table_id: TableId, (offset_option, limit_option): Bounds, index_meta: IndexMetaRef, - ranges: R, + reverse: bool, with_pk: bool, deserializers: &'a TupleDeserializers, covered: bool, - cover_mapping_indices: Option<&[usize]>, - ) -> Result, DatabaseError> - where - R: Into>, - { - let meta = arena.index(index_meta); - let (inner, cover_mapping) = if covered && !matches!(meta.ty, IndexType::PrimaryKey { .. }) - { - let tuple_len = match &meta.value_ty { - LogicalType::Tuple(tys) => tys.len(), - _ => 1, - }; - let columns_len = deserializers.read_columns().count(); - let cover_mapping = match cover_mapping_indices { - Some(slots) => Some(TupleMapping::new(slots.iter().copied(), tuple_len)), - None if columns_len < tuple_len => { - // Omitted mapping denotes an ordered prefix, not every key field. - Some(TupleMapping::new(0..columns_len, tuple_len)) - } - None => None, - }; - (IndexImplEnum::Covered(CoveredIndexImpl), cover_mapping) - } else { - (IndexImplEnum::instance(meta.ty), None) - }; - let ranges = ranges.into(); + ) -> Result, DatabaseError> { + let inner = IndexImplEnum::instance(arena.index(index_meta).ty, covered); Ok(IndexIter { bounds: IterBounds::new(offset_option.unwrap_or(0), limit_option), @@ -302,15 +278,12 @@ pub trait StorageTransaction: Sized { deserializers, tx: self, cursor: None, - cover_mapping, with_pk, - reverse: ranges.reverse, + reverse, }, inner, - ranges, + range_pos: 0, state: IndexIterState::Init, - encode_min_buffer: Bytes::new(), - encode_max_buffer: Bytes::new(), }) } @@ -1366,6 +1339,7 @@ trait IndexImpl { key: &[u8], value: &[u8], params: &IndexImplParams, + cover_mapping: Option<&TupleMapping>, ) -> Result<(), DatabaseError>; fn eq_to_res<'a>( @@ -1397,9 +1371,10 @@ enum IndexImplEnum { } impl IndexImplEnum { - fn instance(index_type: IndexType) -> IndexImplEnum { + fn instance(index_type: IndexType, covered: bool) -> IndexImplEnum { match index_type { IndexType::PrimaryKey { .. } => IndexImplEnum::PrimaryKey(PrimaryKeyIndexImpl), + _ if covered => IndexImplEnum::Covered(CoveredIndexImpl), IndexType::Unique => IndexImplEnum::Unique(UniqueIndexImpl), IndexType::Normal => IndexImplEnum::Normal(NormalIndexImpl), IndexType::Composite => IndexImplEnum::Composite(CompositeIndexImpl), @@ -1413,12 +1388,34 @@ struct NormalIndexImpl; struct CompositeIndexImpl; struct CoveredIndexImpl; -struct TupleMapping { +pub(crate) struct TupleMapping { index_to_scan: Vec, target_len: usize, } impl TupleMapping { + pub(crate) fn cover( + meta: &IndexMeta, + deserializers: &TupleDeserializers, + indices: Option<&[usize]>, + ) -> Option { + if matches!(meta.ty, IndexType::PrimaryKey { .. }) { + return None; + } + let tuple_len = match &meta.value_ty { + LogicalType::Tuple(tys) => tys.len(), + _ => 1, + }; + let columns_len = deserializers.read_columns().count(); + match indices { + Some(slots) => Some(TupleMapping::new(slots.iter().copied(), tuple_len)), + None if columns_len < tuple_len => { + Some(TupleMapping::new(0..columns_len, tuple_len)) + } + None => None, + } + } + fn new(scan_to_index: impl Iterator, tuple_len: usize) -> Self { let mut index_to_scan = vec![usize::MAX; tuple_len]; let mut target_len = 0; @@ -1453,7 +1450,6 @@ struct IndexImplParams<'a, T: StorageTransaction> { deserializers: &'a TupleDeserializers, tx: &'a T, cursor: Option>, - cover_mapping: Option, with_pk: bool, reverse: bool, } @@ -1548,22 +1544,23 @@ impl IndexImpl for IndexImplEnum { key: &[u8], value: &[u8], params: &IndexImplParams, + cover_mapping: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { match self { IndexImplEnum::PrimaryKey(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params) + inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) } IndexImplEnum::Unique(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params) + inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) } IndexImplEnum::Normal(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params) + inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) } IndexImplEnum::Composite(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params) + inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) } IndexImplEnum::Covered(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params) + inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) } } } @@ -1632,6 +1629,7 @@ impl IndexImpl for PrimaryKeyIndexImpl { key: &[u8], value: &[u8], params: &IndexImplParams, + _: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { let tuple_id = params .with_pk @@ -1728,6 +1726,7 @@ impl IndexImpl for UniqueIndexImpl { _: &[u8], value: &[u8], params: &IndexImplParams, + _: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { secondary_index_lookup(table_codec, tuple, value, params) } @@ -1787,6 +1786,7 @@ impl IndexImpl for NormalIndexImpl { _: &[u8], value: &[u8], params: &IndexImplParams, + _: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { secondary_index_lookup(table_codec, tuple, value, params) } @@ -1835,6 +1835,7 @@ impl IndexImpl for CompositeIndexImpl { _: &[u8], value: &[u8], params: &IndexImplParams, + _: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { secondary_index_lookup(table_codec, tuple, value, params) } @@ -1887,9 +1888,10 @@ impl IndexImpl for CoveredIndexImpl { key: &[u8], value: &[u8], params: &IndexImplParams, + cover_mapping: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { let ty = params.value_ty(); - match ¶ms.cover_mapping { + match cover_mapping { Some(mapping) => TableCodec::decode_index_into( tuple, key, @@ -2121,78 +2123,6 @@ impl<'a, T: StorageTransaction + 'a> Iter for TupleIter<'a, T> { } } -enum RangeSource<'r> { - Borrowed(&'r [Range]), - Owned(Range), -} - -pub struct IndexRanges<'r> { - source: RangeSource<'r>, - next: usize, - reverse: bool, -} - -impl IndexRanges<'_> { - pub(crate) fn reversed(mut self, reverse: bool) -> Self { - self.reverse = reverse; - self - } - - pub(crate) fn next(&mut self) -> Option<&Range> { - let ranges = match &self.source { - RangeSource::Borrowed(ranges) => ranges, - RangeSource::Owned(Range::SortedRanges(ranges)) => ranges.as_slice(), - RangeSource::Owned(range) => std::slice::from_ref(range), - }; - let idx = if self.reverse { - ranges.len().checked_sub(self.next + 1)? - } else { - self.next - }; - let range = ranges.get(idx)?; - self.next += 1; - Some(range) - } -} - -impl<'r> From<&'r [Range]> for IndexRanges<'r> { - fn from(ranges: &'r [Range]) -> Self { - IndexRanges { - source: RangeSource::Borrowed(ranges), - next: 0, - reverse: false, - } - } -} - -impl<'r> From<&'r Range> for IndexRanges<'r> { - fn from(range: &'r Range) -> Self { - match range { - Range::SortedRanges(ranges) => ranges.as_slice().into(), - range => std::slice::from_ref(range).into(), - } - } -} - -impl From for IndexRanges<'_> { - fn from(range: Range) -> Self { - IndexRanges { - source: RangeSource::Owned(range), - next: 0, - reverse: false, - } - } -} - -impl<'r> From> for IndexRanges<'r> { - fn from(range: Cow<'r, Range>) -> Self { - match range { - Cow::Borrowed(range) => range.into(), - Cow::Owned(range) => range.into(), - } - } -} - #[derive(Debug, Clone, Copy)] struct IterBounds { offset: usize, @@ -2224,14 +2154,18 @@ impl IterBounds { } } +#[derive(Clone, Copy)] +pub(crate) struct IndexScanView<'s> { + pub(crate) range: &'s Range, + pub(crate) cover_mapping: Option<&'s TupleMapping>, +} + pub struct IndexIter<'a, T: StorageTransaction> { bounds: IterBounds, params: IndexImplParams<'a, T>, inner: IndexImplEnum, - ranges: IndexRanges<'a>, + range_pos: usize, state: IndexIterState<'a, T>, - encode_min_buffer: Bytes, - encode_max_buffer: Bytes, } pub enum IndexIterState<'a, T: StorageTransaction + 'a> { @@ -2241,25 +2175,7 @@ pub enum IndexIterState<'a, T: StorageTransaction + 'a> { } impl<'a, T: StorageTransaction> IndexIter<'a, T> { - pub(crate) fn reset(&mut self, ranges: IndexRanges<'a>, bounds: Bounds) { - self.params.reverse = ranges.reverse; - self.ranges = ranges; - self.rewind(bounds); - } - - pub(crate) fn reset_probe(&mut self, probe: RuntimeIndexProbe, bounds: Bounds) { - match (&mut self.ranges.source, probe) { - (RangeSource::Owned(Range::Eq(value)), RuntimeIndexProbe::Eq(probe)) => { - *value = RangeValue::Value(probe) - } - (source, probe) => *source = RangeSource::Owned(probe.into()), - } - self.rewind(bounds); - } - pub(crate) fn recycle(self, pool: &mut BytesPool) { - pool.recycle_buffer(self.encode_min_buffer); - pool.recycle_buffer(self.encode_max_buffer); if let IndexIterState::Range(iter) = self.state { iter.recycle(pool); } @@ -2267,14 +2183,26 @@ impl<'a, T: StorageTransaction> IndexIter<'a, T> { pub(crate) fn rewind(&mut self, bounds: Bounds) { self.state = IndexIterState::Init; - self.ranges.next = 0; + self.range_pos = 0; self.bounds = IterBounds::new(bounds.0.unwrap_or(0), bounds.1); } -} -impl Iter for IndexIter<'_, T> { - fn next_tuple_into( + fn next_range<'r>(&mut self, range: &'r Range) -> Option<&'r Range> { + let ranges = range.as_slice(); + let idx = if self.params.reverse { + ranges.len().checked_sub(self.range_pos + 1)? + } else { + self.range_pos + }; + let range = ranges.get(idx)?; + self.range_pos += 1; + Some(range) + } + + pub(crate) fn next_tuple_into( &mut self, + view: IndexScanView<'_>, + bound_buffers: &mut [Bytes; 2], table_codec: &mut TableCodec, tuple: &mut Tuple, ) -> Result { @@ -2287,13 +2215,7 @@ impl Iter for IndexIter<'_, T> { loop { match &mut self.state { IndexIterState::Init => { - // Encode bounds into buffers that earlier iterators handed back. - for buffer in [&mut self.encode_min_buffer, &mut self.encode_max_buffer] { - if buffer.capacity() == 0 { - *buffer = table_codec.bytes_pool().buffer(); - } - } - let Some(binary) = self.ranges.next() else { + let Some(binary) = self.next_range(view.range) else { self.state = IndexIterState::Over; continue; }; @@ -2301,11 +2223,12 @@ impl Iter for IndexIter<'_, T> { Range::Scope { min, max } => { let table_id = self.params.table_id; let index_meta = self.params.index_meta(); + let [encode_min, encode_max] = &mut *bound_buffers; let encode_min = encode_bound( table_codec, min, false, - &mut self.encode_min_buffer, + encode_min, &self.params, &self.inner, )?; @@ -2313,7 +2236,7 @@ impl Iter for IndexIter<'_, T> { table_codec, max, true, - &mut self.encode_max_buffer, + encode_max, &self.params, &self.inner, )?; @@ -2338,6 +2261,7 @@ impl Iter for IndexIter<'_, T> { self.state = IndexIterState::Range(iter); } Range::Eq(val) => { + let [encode_min, encode_max] = &mut *bound_buffers; if self.params.cursor.is_none() && matches!(self.inner, IndexImplEnum::PrimaryKey(_)) { @@ -2348,8 +2272,8 @@ impl Iter for IndexIter<'_, T> { tuple, val.value()?, &self.params, - &mut self.encode_min_buffer, - &mut self.encode_max_buffer, + encode_min, + encode_max, )? { IndexResult::Hit => { if self.bounds.consume_offset() { @@ -2379,6 +2303,7 @@ impl Iter for IndexIter<'_, T> { key, value, &self.params, + view.cover_mapping, )?; return Ok(true); @@ -2632,15 +2557,14 @@ pub(crate) fn deserializers_for_test( } } +/// Reads one tuple with `next`, which is given a fresh codec and tuple. #[cfg(test)] -pub(crate) fn next_tuple_for_test(iter: &mut I) -> Result, DatabaseError> { +pub(crate) fn next_tuple_for_test( + mut next: impl FnMut(&mut TableCodec, &mut Tuple) -> Result, +) -> Result, DatabaseError> { let mut tuple = Tuple::default(); let mut table_codec = TableCodec::default(); - if iter.next_tuple_into(&mut table_codec, &mut tuple)? { - Ok(Some(tuple)) - } else { - Ok(None) - } + Ok(next(&mut table_codec, &mut tuple)?.then_some(tuple)) } #[cfg(all(test, not(target_arch = "wasm32")))] @@ -2658,8 +2582,8 @@ mod test { use crate::storage::table_codec::TableCodec; use crate::storage::NameMapping; use crate::storage::{ - project_deserializers_for_test, IndexIter, InnerIter, Storage, StorageTransaction, - TableCache, TupleDeserializers, + project_deserializers_for_test, IndexIter, IndexScanView, InnerIter, Iter, Storage, + StorageTransaction, TableCache, TupleDeserializers, }; use crate::types::index::{Index, IndexType}; use crate::types::tuple::Tuple; @@ -2873,15 +2797,15 @@ mod test { )?; assert_eq!( - super::next_tuple_for_test(&mut tuple_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), tuples[0] ); assert_eq!( - super::next_tuple_for_test(&mut tuple_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), tuples[1] ); assert_eq!( - super::next_tuple_for_test(&mut tuple_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), tuples[2] ); @@ -2916,11 +2840,11 @@ mod test { )?; assert_eq!( - super::next_tuple_for_test(&mut tuple_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), tuples[0] ); assert_eq!( - super::next_tuple_for_test(&mut tuple_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), tuples[2] ); @@ -3093,16 +3017,16 @@ mod test { names.table(table_cache, "t1").unwrap().id(), (None, None), index_meta, - &[Range::Scope { - min: Bound::Unbounded, - max: Bound::Unbounded, - }][..], + false, true, deserializers, false, - None, ) } + let full = Range::Scope { + min: Bound::Unbounded, + max: Bound::Unbounded, + }; let mut table_codec = TableCodec::default(); let temp_dir = TempDir::new().expect("unable to create temporary working directory"); @@ -3196,15 +3120,48 @@ mod test { )?; assert_eq!( - super::next_tuple_for_test(&mut index_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| { + index_iter.next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? + .unwrap(), tuples[0] ); assert_eq!( - super::next_tuple_for_test(&mut index_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| { + index_iter.next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? + .unwrap(), tuples[2] ); assert_eq!( - super::next_tuple_for_test(&mut index_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| { + index_iter.next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? + .unwrap(), tuples[1] ); @@ -3240,11 +3197,33 @@ mod test { )?; assert_eq!( - super::next_tuple_for_test(&mut index_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| { + index_iter.next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? + .unwrap(), tuples[2] ); assert_eq!( - super::next_tuple_for_test(&mut index_iter)?.unwrap(), + super::next_tuple_for_test(|codec, tuple| { + index_iter.next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? + .unwrap(), tuples[1] ); diff --git a/src/storage/rocksdb.rs b/src/storage/rocksdb.rs index 6cda8375..85997a14 100644 --- a/src/storage/rocksdb.rs +++ b/src/storage/rocksdb.rs @@ -800,8 +800,8 @@ mod test { use crate::storage::rocksdb::{OptimisticRocksStorage, RocksStorage}; use crate::storage::table_codec::TableCodec; use crate::storage::{ - IndexImplEnum, IndexImplParams, IndexIter, IndexIterState, InnerIter, IterBounds, - PrimaryKeyIndexImpl, Storage, StorageTransaction, + IndexImplEnum, IndexImplParams, IndexIter, IndexIterState, IndexScanView, InnerIter, Iter, + IterBounds, PrimaryKeyIndexImpl, Storage, StorageTransaction, TupleMapping, }; use crate::types::index::IndexType; use crate::types::serialize::BytesPool; @@ -1031,10 +1031,12 @@ mod test { (Some(1), Some(1)), )?; - let option_1 = crate::storage::next_tuple_for_test(&mut iter)?; + let option_1 = + crate::storage::next_tuple_for_test(|codec, tuple| iter.next_tuple_into(codec, tuple))?; assert_eq!(option_1.unwrap().pk, Some(DataValue::Int32(2))); - let option_2 = crate::storage::next_tuple_for_test(&mut iter)?; + let option_2 = + crate::storage::next_tuple_for_test(|codec, tuple| iter.next_tuple_into(codec, tuple))?; assert_eq!(option_2, None); Ok(()) @@ -1095,19 +1097,27 @@ mod test { deserializers: &deserializers, tx: &transaction, cursor: None, - cover_mapping: None, with_pk: true, reverse: false, }, - ranges: (&ranges[..]).into(), + range_pos: 0, state: IndexIterState::Init, inner: IndexImplEnum::PrimaryKey(PrimaryKeyIndexImpl), - encode_min_buffer: Vec::new(), - encode_max_buffer: Vec::new(), }; + let range = Range::SortedRanges(ranges.to_vec()); let mut result = Vec::new(); - while let Some(tuple) = crate::storage::next_tuple_for_test(&mut iter)? { + while let Some(tuple) = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? { result.push(tuple.pk.unwrap()); } @@ -1141,6 +1151,10 @@ mod test { &plan_arena, true, ); + let range = Range::Scope { + min: Bound::Excluded(DataValue::Int32(0).into()), + max: Bound::Unbounded, + }; let mut iter = transaction .read_by_index( &plan_arena, @@ -1152,18 +1166,24 @@ mod test { .id(), (Some(0), Some(1)), table.indexes[0], - Range::Scope { - min: Bound::Excluded(DataValue::Int32(0).into()), - max: Bound::Unbounded, - }, + false, true, &deserializers, false, - None, ) .unwrap(); - while let Some(tuple) = crate::storage::next_tuple_for_test(&mut iter)? { + while let Some(tuple) = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? { assert_eq!(tuple.pk, Some(DataValue::Int32(1))); assert_eq!(tuple.values, vec![DataValue::Int32(1), DataValue::Int32(1)]) } @@ -1181,6 +1201,10 @@ mod test { &plan_arena, true, ); + let range = Range::Scope { + min: Bound::Excluded(DataValue::Int32(3).into()), + max: Bound::Unbounded, + }; let mut iter = transaction .read_by_index( &plan_arena, @@ -1192,18 +1216,24 @@ mod test { .id(), (Some(0), Some(1)), table.indexes[0], - Range::Scope { - min: Bound::Excluded(DataValue::Int32(3).into()), - max: Bound::Unbounded, - }, + false, true, &deserializers, false, - None, ) .unwrap(); - while let Some(tuple) = crate::storage::next_tuple_for_test(&mut iter)? { + while let Some(tuple) = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + })? { assert_eq!(tuple.pk, Some(DataValue::Int32(3))); assert_eq!(tuple.values, vec![DataValue::Int32(3)]) } @@ -1251,6 +1281,8 @@ mod test { &plan_arena, false, ); + let range = Range::SortedRanges(c1_ranges.to_vec()); + let mapping = TupleMapping::cover(plan_arena.index(idx_c1), &deserializers, None); let mut iter = transaction.read_by_index( &plan_arena, kite_sql @@ -1261,15 +1293,24 @@ mod test { .id(), (None, None), idx_c1, - &c1_ranges[..], + false, false, &deserializers, true, - None, )?; let mut values = Vec::new(); - while let Some(tuple) = crate::storage::next_tuple_for_test(&mut iter)? { + while let Some(tuple) = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: mapping.as_ref(), + }, + &mut Default::default(), + codec, + tuple, + ) + })? { values.push(tuple.values[0].clone()); } assert_eq!( @@ -1355,6 +1396,15 @@ mod test { &plan_arena, false, ); + let range = Range::Scope { + min: Bound::Unbounded, + max: Bound::Unbounded, + }; + let mapping = TupleMapping::cover( + plan_arena.index(composite_index), + &deserializers, + Some(&cover_mapping), + ); let mut iter = transaction.read_by_index( &plan_arena, kite_sql @@ -1365,16 +1415,23 @@ mod test { .id(), (None, None), composite_index, - &[Range::Scope { - min: Bound::Unbounded, - max: Bound::Unbounded, - }][..], + false, false, &deserializers, true, - Some(&cover_mapping), )?; - let first_tuple = crate::storage::next_tuple_for_test(&mut iter)?.unwrap(); + let first_tuple = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: mapping.as_ref(), + }, + &mut Default::default(), + codec, + tuple, + ) + })? + .unwrap(); assert_eq!( first_tuple.values, vec![DataValue::Int32(0), DataValue::Int32(0)] @@ -1391,6 +1448,11 @@ mod test { &plan_arena, false, ); + let range = Range::Scope { + min: Bound::Unbounded, + max: Bound::Unbounded, + }; + let mapping = TupleMapping::cover(plan_arena.index(composite_index), &deserializers, None); let mut iter = transaction.read_by_index( &plan_arena, kite_sql @@ -1401,20 +1463,38 @@ mod test { .id(), (None, None), composite_index, - &[Range::Scope { - min: Bound::Unbounded, - max: Bound::Unbounded, - }][..], + false, false, &deserializers, true, - None, )?; for value in [0, 1, 2, 4] { - let tuple = crate::storage::next_tuple_for_test(&mut iter)?.unwrap(); + let tuple = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: mapping.as_ref(), + }, + &mut Default::default(), + codec, + tuple, + ) + })? + .unwrap(); assert_eq!(tuple.values, vec![DataValue::Int32(value)]); } - assert!(crate::storage::next_tuple_for_test(&mut iter)?.is_none()); + assert!(crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: mapping.as_ref(), + }, + &mut Default::default(), + codec, + tuple, + ) + })? + .is_none()); drop(iter); let target_pk = DataValue::Int32(3); @@ -1431,6 +1511,8 @@ mod test { &plan_arena, false, ); + let range = Range::SortedRanges(covered_ranges.to_vec()); + let mapping = TupleMapping::cover(plan_arena.index(unique_index), &deserializers, None); let mut iter = transaction.read_by_index( &plan_arena, kite_sql @@ -1441,15 +1523,24 @@ mod test { .id(), (Some(0), Some(1)), unique_index, - &covered_ranges[..], + false, false, &deserializers, true, - None, )?; let mut tuples = Vec::new(); - while let Some(tuple) = crate::storage::next_tuple_for_test(&mut iter)? { + while let Some(tuple) = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: mapping.as_ref(), + }, + &mut Default::default(), + codec, + tuple, + ) + })? { tuples.push(tuple); } @@ -1472,6 +1563,11 @@ mod test { &plan_arena, false, ); + let range = Range::Scope { + min: Bound::Unbounded, + max: Bound::Unbounded, + }; + let mapping = TupleMapping::cover(plan_arena.index(pk_index), &deserializers, Some(&[0])); let mut iter = transaction.read_by_index( &plan_arena, kite_sql @@ -1482,17 +1578,23 @@ mod test { .id(), (None, None), pk_index, - &[Range::Scope { - min: Bound::Unbounded, - max: Bound::Unbounded, - }][..], + false, false, &deserializers, true, - Some(&[0]), )?; let mut row_count = 0; - while let Some(tuple) = crate::storage::next_tuple_for_test(&mut iter)? { + while let Some(tuple) = crate::storage::next_tuple_for_test(|codec, tuple| { + iter.next_tuple_into( + IndexScanView { + range: &range, + cover_mapping: mapping.as_ref(), + }, + &mut Default::default(), + codec, + tuple, + ) + })? { assert_eq!(tuple.values.len(), 1); row_count += 1; } diff --git a/src/types/serialize.rs b/src/types/serialize.rs index 4ff92cc2..be8bb3b3 100644 --- a/src/types/serialize.rs +++ b/src/types/serialize.rs @@ -255,10 +255,6 @@ impl BytesPool { } } - pub(crate) fn buffer(&mut self) -> Vec { - self.0.pop().unwrap_or_default() - } - pub(crate) fn recycle_buffer(&mut self, mut buffer: Vec) { if self.0.len() < Self::CAPACITY && buffer.capacity() > 0 diff --git a/src/types/value.rs b/src/types/value.rs index bc7eef4d..43c8a8fd 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -32,9 +32,7 @@ use std::cmp::Ordering; use std::fmt::Formatter; use std::hash::Hash; use std::io::Read; -#[cfg(feature = "decimal")] -use std::mem; -use std::{cmp, fmt, slice}; +use std::{cmp, fmt, mem, slice}; trait ReadByteExt: Read { fn read_u8(&mut self) -> std::io::Result { @@ -722,8 +720,9 @@ impl DataValue { digits.parse().ok() } - pub(crate) fn bind_parameters<'p, F, G>( + pub(crate) fn bind_from<'p, F, G>( &mut self, + template: &DataValue, params: &F, init_values: &G, ) -> Result @@ -731,32 +730,60 @@ impl DataValue { F: Fn(usize) -> Option<&'p DataValue>, G: Fn(ScalarQueryRef) -> Option<&'p DataValue>, { - match self { - DataValue::Parameter { id, ty } => { - let Some(input) = params(*id) else { - return Err(DatabaseError::parameter_not_found(format!("${id}"))); - }; - *self = input.clone().cast(ty)?; - Ok(self.is_null()) - } - DataValue::InitValue { id, ty } => { - let Some(input) = init_values(*id) else { - return Err(DatabaseError::InvalidValue(format!( - "scalar query {id} is not initialized" - ))); + let (input, ty) = match template { + DataValue::Parameter { id, ty } => ( + params(*id).ok_or_else(|| DatabaseError::parameter_not_found(format!("${id}")))?, + ty, + ), + DataValue::InitValue { id, ty } => ( + init_values(*id).ok_or_else(|| { + DatabaseError::InvalidValue(format!("scalar query {id} is not initialized")) + })?, + ty, + ), + DataValue::Tuple(templates) => { + return match self { + DataValue::Tuple(values) => { + Self::bind_values_from(values, templates, params, init_values) + } + _ => { + let mut values = Vec::with_capacity(templates.len()); + let has_null = + Self::bind_values_from(&mut values, templates, params, init_values)?; + *self = DataValue::Tuple(values); + Ok(has_null) + } }; - *self = input.clone().cast(ty)?; - Ok(self.is_null()) } - DataValue::Tuple(values) => { - let mut has_null = false; - for value in values { - has_null |= value.bind_parameters(params, init_values)?; - } - Ok(has_null) + template => { + self.clone_from(template); + return Ok(false); } - _ => Ok(false), + }; + self.clone_from(input); + if self.logical_type() != *ty { + *self = mem::replace(self, DataValue::Null).cast(ty)?; + } + Ok(self.is_null()) + } + + pub(crate) fn bind_values_from<'p, F, G>( + values: &mut Vec, + templates: &[DataValue], + params: &F, + init_values: &G, + ) -> Result + where + F: Fn(usize) -> Option<&'p DataValue>, + G: Fn(ScalarQueryRef) -> Option<&'p DataValue>, + { + values.truncate(templates.len()); + values.resize_with(templates.len(), || DataValue::Null); + let mut has_null = false; + for (value, template) in values.iter_mut().zip(templates) { + has_null |= value.bind_from(template, params, init_values)?; } + Ok(has_null) } pub(crate) fn has_parameter(&self) -> bool { From 0b4e025abff68c3ca5c2d4cb41aa426fc8fe32ac Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 4 Nov 2026 17:16:23 +0800 Subject: [PATCH 05/14] perf(program): reuse iterator slots and bind borrowed parameters - Executions reuse the allocation of their storage iterator slots: an emptied slot Vec goes back to ExecBuffer type-erased (SpareIters) and comes back when the slot layout matches. New slots reserve exactly. - A ProgramIter dropped before its rows run out still hands its iterator buffers and slots back to the program. - Program::execute takes `(usize, Param)` pairs. Param::Str and Param::Bytes borrow, and binding copies them into the buffers of the previous execution, so string parameters bind without allocating. tpcc's DbParam follows. --- examples/program.rs | 9 +-- src/db.rs | 34 +++++----- src/db/program.rs | 124 ++++++++++++++++++++++++++++--------- src/execution/mod.rs | 90 +++++++++++++++++++++++---- src/orm/mod.rs | 4 +- src/planner/arena.rs | 38 ++++++++++-- src/storage/mod.rs | 4 +- tpcc/src/backend/mod.rs | 5 +- tpcc/src/backend/sqlite.rs | 10 ++- tpcc/src/delivery.rs | 46 +++++++------- tpcc/src/new_ord.rs | 76 +++++++++++------------ tpcc/src/order_stat.rs | 36 +++++------ tpcc/src/payment.rs | 85 +++++++++++++------------ tpcc/src/slev.rs | 18 +++--- 14 files changed, 371 insertions(+), 208 deletions(-) diff --git a/examples/program.rs b/examples/program.rs index 80ccb0e8..312ffd28 100644 --- a/examples/program.rs +++ b/examples/program.rs @@ -65,13 +65,14 @@ mod app { .execute( &mut transaction, [ - (1, DataValue::Int32(c1)), - (2, DataValue::Int32(c1 * 10 * round)), + (1, DataValue::Int32(c1).into()), + (2, DataValue::Int32(c1 * 10 * round).into()), ], )? .done()?; - let mut iter = select.execute(&mut transaction, [(1, DataValue::Int32(c1))])?; + let mut iter = + select.execute(&mut transaction, [(1, DataValue::Int32(c1).into())])?; let c2 = iter.next_tuple(|_, tuple| tuple[0].i32())?.unwrap(); assert_eq!(c2, Some(c1 * 10 * round)); } @@ -80,7 +81,7 @@ mod app { // A later transaction sees what the earlier ones committed. let mut transaction = database.new_transaction(true)?; - let mut iter = select.execute(&mut transaction, [(1, DataValue::Int32(2))])?; + let mut iter = select.execute(&mut transaction, [(1, DataValue::Int32(2).into())])?; let c2 = iter.next_tuple(|_, tuple| tuple[0].i32())?.unwrap(); assert_eq!(c2, Some(40)); drop(iter); diff --git a/src/db.rs b/src/db.rs index 38a626a8..f0ab6b51 100644 --- a/src/db.rs +++ b/src/db.rs @@ -18,7 +18,7 @@ pub use crate::binder::Statement; use crate::binder::{Binder, BinderContext}; use crate::errors::DatabaseError; pub use crate::execution::Executor; -use crate::execution::{DDLApply, ExecArena}; +use crate::execution::{DDLApply, ExecArena, ExecBuffer}; use crate::expression::function::scala::ScalarFunctionImpl; use crate::expression::function::table::{ ArcTableFunctionImpl, TableFunctionCatalog, TableFunctionImpl, @@ -54,12 +54,11 @@ use crate::storage::{ CheckpointableStorage, NameMapping, StatisticsMetaCache, Storage, StorageTransaction, TableCache, TransactionIsolationLevel, ViewCache, }; -use crate::types::serialize::BytesPool; use crate::types::tuple::{Schema, SchemaView, TupleImpl}; use crate::types::value::DataValue; use crate::types::LogicalType; use ordered_float::OrderedFloat; -pub use program::{Program, ProgramIter}; +pub use program::{Param, Program, ProgramIter}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; @@ -629,7 +628,8 @@ fn start_execution_inner<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p>( plan_arena: &mut ExecMetaArena, ) -> Result, DatabaseError> { let stamp = statement_stamp(transaction, executor.plan())?; - let mut arena = ExecArena::new(table_cache, transaction).with_iters(executor.iters()); + let iters = executor.buffer.iter_slots(executor.iters()); + let mut arena = ExecArena::new(table_cache, transaction).with_iters(iters); executor.buffer.set_statement_stamp(stamp); executor.start(&mut arena, plan_arena)?; @@ -1563,12 +1563,12 @@ struct TransactionGuard<'a, 'p, T: StorageTransaction + 'p> { impl TransactionGuard<'_, '_, T> { #[inline] - fn finish(&mut self, pool: Option<&mut BytesPool>) { + fn finish(&mut self, buffer: Option<&mut ExecBuffer>) { if !self.active { return; } self.active = false; - self.arena.release_iters(pool); + self.arena.release_iters(buffer); } } @@ -1596,11 +1596,11 @@ impl<'p, T: StorageTransaction + 'p> TransactionGuard<'_, 'p, T> { ))) } Ok(false) => { - self.finish(Some(executor.buffer.bytes_pool())); + self.finish(Some(&mut executor.buffer)); Ok(None) } Err(err) => { - self.finish(Some(executor.buffer.bytes_pool())); + self.finish(Some(&mut executor.buffer)); Err(err) } } @@ -2400,7 +2400,7 @@ pub(crate) mod test { &[(1, LogicalType::Integer)], )?; - let mut iter = statement.execute(&mut tx, &[(1, DataValue::Int32(0))])?; + let mut iter = statement.execute(&mut tx, &[(1, DataValue::Int32(0).into())])?; let row = next_tuple_owned(&mut iter)?.unwrap(); let plan = row[0].utf8().unwrap(); @@ -2423,10 +2423,10 @@ pub(crate) mod test { let mut iter = statement.execute( &mut tx, &[ - (1, DataValue::Int32(0)), - (2, DataValue::Int32(0)), - (3, DataValue::Int32(1)), - (4, DataValue::Int32(0)), + (1, DataValue::Int32(0).into()), + (2, DataValue::Int32(0).into()), + (3, DataValue::Int32(1).into()), + (4, DataValue::Int32(0).into()), ], )?; let row = next_tuple_owned(&mut iter)?.unwrap(); @@ -2442,10 +2442,10 @@ pub(crate) mod test { let mut iter = statement.execute( &mut tx, &[ - (1, DataValue::Int32(9)), - (2, DataValue::Int32(0)), - (3, DataValue::Int32(1)), - (4, DataValue::Int32(0)), + (1, DataValue::Int32(9).into()), + (2, DataValue::Int32(0).into()), + (3, DataValue::Int32(1).into()), + (4, DataValue::Int32(0).into()), ], )?; let row = next_tuple_owned(&mut iter)?.unwrap(); diff --git a/src/db/program.rs b/src/db/program.rs index 5357d50d..3fee9e76 100644 --- a/src/db/program.rs +++ b/src/db/program.rs @@ -24,6 +24,41 @@ use crate::types::index::IndexLookup; // GRCOV_EXCL_STOP use crate::types::LogicalType; +#[derive(Debug, Clone)] +pub enum Param<'a> { + Value(DataValue), + Str(&'a str), + Bytes(&'a [u8]), +} + +impl Param<'_> { + pub(crate) fn to_value(&self) -> DataValue { + match self { + Param::Value(value) => value.clone(), + Param::Str(value) => DataValue::from(value.to_string()), + Param::Bytes(value) => DataValue::from(value.to_vec()), + } + } +} + +impl From for Param<'_> { + fn from(value: DataValue) -> Self { + Param::Value(value) + } +} + +impl<'a> From<&'a str> for Param<'a> { + fn from(value: &'a str) -> Self { + Param::Str(value) + } +} + +impl<'a> From<&'a [u8]> for Param<'a> { + fn from(value: &'a [u8]) -> Self { + Param::Bytes(value) + } +} + /// A bound and optimized plan with its executor tree, reused by every execution. pub struct Program<'p> { pub(crate) executor: Executor<'p>, @@ -33,10 +68,10 @@ pub struct Program<'p> { impl<'p> Program<'p> { /// Runs the program with `params` in `transaction`. The returned iterator borrows the /// program until the execution ends. - pub fn execute<'a, 'txn, S: Storage>( + pub fn execute<'a, 'txn, 'v, S: Storage>( &'a mut self, transaction: &'a mut Transaction<'txn, S>, - params: impl AsRef<[(usize, DataValue)]>, + params: impl AsRef<[(usize, Param<'v>)]>, ) -> Result>, DatabaseError> where 'txn: 'p, @@ -66,7 +101,7 @@ impl<'p> Program<'p> { fn bind( &mut self, table_arena: &TableArenaCell, - params: &[(usize, DataValue)], + params: &[(usize, Param<'_>)], ) -> Result<(), DatabaseError> { if !std::ptr::eq( self.arena.parent().plan_arena().table_arena_cell(), @@ -194,6 +229,12 @@ impl<'a, 'p, T: StorageTransaction + 'p> ProgramIter<'a, 'p, T> { } } +impl Drop for ProgramIter<'_, '_, T> { + fn drop(&mut self) { + self.guard.finish(Some(&mut self.program.executor.buffer)); + } +} + impl<'a, 'p, T: StorageTransaction + 'p> TxnResultIter<'a> for ProgramIter<'a, 'p, T> { fn next_txn_tuple( &mut self, @@ -247,7 +288,7 @@ mod tests { )?; let mut tx = db.new_transaction(false)?; for (value, expected) in [(7, 31_i64), (23, 87_i64)] { - let mut iter = plan.execute(&mut tx, [(1, DataValue::Int64(value))])?; + let mut iter = plan.execute(&mut tx, [(1, DataValue::Int64(value).into())])?; iter.schema(|schema| { assert_eq!(schema.len(), 1); assert_eq!( @@ -262,7 +303,7 @@ mod tests { assert!(iter.next_tuple(|_, _| ())?.is_none()); iter.done()?; } - let mut iter = plan.execute(&mut tx, [(1, DataValue::Int64(7))])?; + let mut iter = plan.execute(&mut tx, [(1, DataValue::Int64(7).into())])?; assert_eq!( iter.next_tuple(|_, row| row.clone())?.unwrap(), vec![DataValue::Int64(31)] @@ -490,7 +531,10 @@ mod tests { assert!(has_filter(lower_bounds.executor.plan())); let mut iter = lower_bounds.execute( &mut tx, - [(1, DataValue::Int32(3)), (2, DataValue::Int32(1))], + [ + (1, DataValue::Int32(3).into()), + (2, DataValue::Int32(1).into()), + ], )?; let mut rows = Vec::new(); while iter @@ -506,7 +550,10 @@ mod tests { )?; let mut iter = equalities.execute( &mut tx, - [(1, DataValue::Int32(2)), (2, DataValue::Int32(2))], + [ + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(2).into()), + ], )?; assert_eq!( iter.next_tuple(|_, row| row.value_at(0).clone())?, @@ -517,7 +564,10 @@ mod tests { let mut iter = equalities.execute( &mut tx, - [(1, DataValue::Int32(2)), (2, DataValue::Int32(3))], + [ + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(3).into()), + ], )?; assert!(iter.next_tuple(|_, _| ())?.is_none()); iter.done()?; @@ -562,7 +612,10 @@ mod tests { vec![], ), ] { - let mut iter = unions.execute(&mut tx, [(1, first), (2, second), (3, lower)])?; + let mut iter = unions.execute( + &mut tx, + [(1, first.into()), (2, second.into()), (3, lower.into())], + )?; let mut rows = Vec::new(); while iter .next_tuple(|_, row| rows.push(row.value_at(0).clone()))? @@ -586,8 +639,10 @@ mod tests { (DataValue::Null, DataValue::Null), (DataValue::Int32(3), DataValue::Int32(13)), ] { - let mut iter = - plan.execute(&mut tx, [(1, DataValue::Int32(10)), (2, value.clone())])?; + let mut iter = plan.execute( + &mut tx, + [(1, DataValue::Int32(10).into()), (2, value.clone().into())], + )?; assert_eq!( iter.next_tuple(|_, row| row.value_at(0).clone())?, Some(expected) @@ -601,12 +656,15 @@ mod tests { } assert!(matches!( - plan.execute(&mut tx, [(1, DataValue::Int32(10))]), + plan.execute(&mut tx, [(1, DataValue::Int32(10).into())]), Err(DatabaseError::ParametersNotFound { name, .. }) if name == "$2" )); let mut iter = plan.execute( &mut tx, - [(1, DataValue::Int32(10)), (2, DataValue::Int32(4))], + [ + (1, DataValue::Int32(10).into()), + (2, DataValue::Int32(4).into()), + ], )?; assert_eq!( iter.next_tuple(|_, row| row.value_at(0).clone())?, @@ -647,13 +705,19 @@ mod tests { ( "select k from t where w = $1 and k < $2 and k >= $3", vec![ - (1, DataValue::Int32(2)), - (2, DataValue::Int32(5)), - (3, DataValue::Null), + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(5).into()), + (3, DataValue::Null.into()), ], ), - ("select k from t where w < $1", vec![(1, DataValue::Null)]), - ("select k from t where w = $1", vec![(1, DataValue::Null)]), + ( + "select k from t where w < $1", + vec![(1, DataValue::Null.into())], + ), + ( + "select k from t where w = $1", + vec![(1, DataValue::Null.into())], + ), ] { let mut plan = db.program_sql( sql, @@ -692,9 +756,9 @@ mod tests { let mut iter = plan.execute( &mut tx, [ - (1, DataValue::Int32(2)), - (2, DataValue::Int32(2)), - (3, DataValue::Int32(5)), + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(2).into()), + (3, DataValue::Int32(5).into()), ], )?; let mut rows = Vec::new(); @@ -707,9 +771,9 @@ mod tests { let mut iter = plan.execute( &mut tx, [ - (1, DataValue::Int32(1)), - (2, DataValue::Int32(1)), - (3, DataValue::Int32(2)), + (1, DataValue::Int32(1).into()), + (2, DataValue::Int32(1).into()), + (3, DataValue::Int32(2).into()), ], )?; assert_eq!( @@ -731,9 +795,9 @@ mod tests { ], )?; plan.arena.parent_mut().bind(&[ - (1, DataValue::Int32(2)), - (2, DataValue::Int32(5)), - (3, DataValue::Int32(22)), + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(5).into()), + (3, DataValue::Int32(22).into()), ])?; fn find_scan(plan: &LogicalPlan) -> Option<&LogicalPlan> { matches!(plan.operator, Operator::TableScan(_)) @@ -809,9 +873,9 @@ mod tests { let mut iter = plan.execute( &mut tx, [ - (1, DataValue::Int32(2)), - (2, DataValue::Int32(5)), - (3, lower), + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(5).into()), + (3, lower.into()), ], )?; let mut rows = Vec::new(); diff --git a/src/execution/mod.rs b/src/execution/mod.rs index ebea95dc..38e74505 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -85,9 +85,11 @@ use crate::types::index::RuntimeIndexProbe; use crate::types::serialize::BytesPool; use crate::types::tuple::{ConcatPool, Input, Schema, Tuple, TupleId, TupleImpl}; use crate::types::value::DataValue; +use std::alloc::{dealloc, Layout}; use std::borrow::Cow; use std::marker::PhantomData; -use std::mem; +use std::mem::{self, ManuallyDrop}; +use std::ptr::NonNull; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum ExecStatus { @@ -584,6 +586,58 @@ pub(crate) struct ExecBuffer { runtime_probe_stack: Vec, restarting: bool, bound_buffers: [Bytes; 2], + spare_iters: Option, +} + +/// The emptied allocation of an execution's storage iterator slots, kept for the next execution. +/// It is untyped because the slots borrow the transaction an execution reads. +struct SpareIters { + ptr: NonNull, + capacity: usize, + slot: Layout, +} + +unsafe impl Send for SpareIters {} +unsafe impl Sync for SpareIters {} + +impl SpareIters { + fn new(iters: Vec) -> Option { + debug_assert!(iters.is_empty()); + if iters.capacity() == 0 || size_of::() == 0 { + return None; + } + let mut iters = ManuallyDrop::new(iters); + Some(SpareIters { + ptr: NonNull::new(iters.as_mut_ptr())?.cast(), + capacity: iters.capacity(), + slot: Layout::new::(), + }) + } + + fn into_vec(self) -> Option> { + if self.slot != Layout::new::() { + return None; + } + let spare = ManuallyDrop::new(self); + // SAFETY: a `Vec` of an element type with the layout of `E` allocated `ptr` for + // `capacity` elements, and none of them is initialized. + Some(unsafe { Vec::from_raw_parts(spare.ptr.as_ptr().cast(), 0, spare.capacity) }) + } +} + +impl Drop for SpareIters { + fn drop(&mut self) { + // SAFETY: the `Vec` that allocated `ptr` did so with this layout. + unsafe { + dealloc( + self.ptr.as_ptr(), + Layout::from_size_align_unchecked( + self.slot.size() * self.capacity, + self.slot.align(), + ), + ) + } + } } pub(crate) struct ExecArena<'a, 't, T: StorageTransaction + 'a> { @@ -714,8 +768,18 @@ impl ExecBuffer { &mut self.result.tuple } - pub(crate) fn bytes_pool(&mut self) -> &mut BytesPool { - self.table_codec.bytes_pool() + pub(crate) fn iter_slots<'t, T: StorageTransaction + 't>( + &mut self, + len: usize, + ) -> Vec> { + let mut iters = self + .spare_iters + .take() + .and_then(SpareIters::into_vec) + .unwrap_or_default(); + iters.reserve_exact(len); + iters.resize_with(len, || StorageIter::Empty); + iters } } @@ -731,19 +795,21 @@ impl<'a, 't, T: StorageTransaction + 'a> ExecArena<'a, 't, T> { } } - pub(crate) fn with_iters(mut self, iters: usize) -> Self { - self.iters.resize_with(iters, || StorageIter::Empty); + pub(crate) fn with_iters(mut self, iters: Vec>) -> Self { + self.iters = iters; self } - /// Drops every storage iterator, handing their buffers back to `pool` when there is one. - pub(crate) fn release_iters(&mut self, mut pool: Option<&mut BytesPool>) { - for iter in &mut self.iters { - let iter = mem::replace(iter, StorageIter::Empty); - if let Some(pool) = pool.as_deref_mut() { - iter.recycle(pool); - } + pub(crate) fn release_iters(&mut self, buffer: Option<&mut ExecBuffer>) { + let Some(buffer) = buffer else { + self.iters.fill_with(|| StorageIter::Empty); + return; + }; + let pool = buffer.table_codec.bytes_pool(); + for iter in self.iters.drain(..) { + iter.recycle(pool); } + buffer.spare_iters = SpareIters::new(mem::take(&mut self.iters)); } pub(crate) fn take_ddl_apply(&mut self) -> Vec { diff --git a/src/orm/mod.rs b/src/orm/mod.rs index 1ad7e967..595cac1f 100644 --- a/src/orm/mod.rs +++ b/src/orm/mod.rs @@ -3968,7 +3968,7 @@ mod tests { let mut tx = database.new_transaction(false)?; for (id, expected) in [(1, "Alice"), (3, "Cara"), (2, "Bob")] { let rows = plan - .execute(&mut tx, [(1, DataValue::Int32(id))])? + .execute(&mut tx, [(1, DataValue::Int32(id).into())])? .orm::() .collect::, _>>()?; assert_eq!(rows.len(), 1); @@ -3980,7 +3980,7 @@ mod tests { .finish() })?; let rows = tx_plan - .execute(&mut tx, [(1, DataValue::Int32(2))])? + .execute(&mut tx, [(1, DataValue::Int32(2).into())])? .orm::() .collect::, _>>()?; assert_eq!(rows[0].name, "Bob"); diff --git a/src/planner/arena.rs b/src/planner/arena.rs index 6173c3ff..e6563f6f 100644 --- a/src/planner/arena.rs +++ b/src/planner/arena.rs @@ -14,6 +14,7 @@ use crate::catalog::view::View; use crate::catalog::{ColumnCatalog, ColumnRef, SourceId, TableCatalog, TableId, TableName}; +use crate::db::Param; use crate::errors::DatabaseError; use crate::expression::visitor_mut::ExprVisitorMut; use crate::expression::ScalarExpression; @@ -26,8 +27,8 @@ use crate::planner::{LogicalPlan, ScalarQueryRef}; use crate::storage::{NameMapping, TableCache, ViewCache}; use crate::types::index::{IndexMeta, IndexMetaRef}; use crate::types::tuple::Schema; -use crate::types::value::DataValue; -use crate::types::LogicalType; +use crate::types::value::{DataValue, Utf8Type}; +use crate::types::{CharLengthUnits, LogicalType}; use std::cell::UnsafeCell; use std::collections::HashSet; use std::fmt; @@ -1206,7 +1207,7 @@ impl<'a> ParamArena<'a> { } } - pub(crate) fn bind(&mut self, params: &[(usize, DataValue)]) -> Result<(), DatabaseError> { + pub(crate) fn bind(&mut self, params: &[(usize, Param<'_>)]) -> Result<(), DatabaseError> { let parent = &self.parent; // Keep the parameter slots of the previous execution and overwrite their values. self.expressions.truncate(self.parameter_count); @@ -1236,12 +1237,33 @@ impl<'a> ParamArena<'a> { param: None, } => { // Overwrite the previous execution's value, reusing its buffer. - value.clone_from(source); + match (&mut *value, source) { + (value, Param::Value(source)) => value.clone_from(source), + ( + DataValue::Utf8 { + value, + ty: utf8_ty, + unit, + }, + Param::Str(source), + ) => { + (*source).clone_into(value.to_mut()); + *utf8_ty = Utf8Type::Variable(None); + *unit = CharLengthUnits::Characters; + } + (DataValue::Binary { value, len }, Param::Bytes(source)) => { + (*source).clone_into(value.to_mut()); + *len = None; + } + (value, source) => *value = source.to_value(), + } if value.logical_type() != *ty { *value = mem::replace(value, DataValue::Null).cast(ty)?; } } - _ => *expr = ArenaExpr::new(ScalarExpression::Constant(source.clone().cast(ty)?)), + _ => { + *expr = ArenaExpr::new(ScalarExpression::Constant(source.to_value().cast(ty)?)) + } } } Ok(()) @@ -1366,8 +1388,12 @@ mod tests { parameter_expressions: &[ExprRef], params: &[(usize, DataValue)], ) -> Result, DatabaseError> { + let params = params + .iter() + .map(|(id, value)| (*id, Param::Value(value.clone()))) + .collect::>(); let mut arena = ParamArena::new(parent.clone(), parameter_expressions.to_vec()); - arena.bind(params)?; + arena.bind(¶ms)?; Ok(arena) } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 6f6dfd1b..e920c3e4 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1409,9 +1409,7 @@ impl TupleMapping { let columns_len = deserializers.read_columns().count(); match indices { Some(slots) => Some(TupleMapping::new(slots.iter().copied(), tuple_len)), - None if columns_len < tuple_len => { - Some(TupleMapping::new(0..columns_len, tuple_len)) - } + None if columns_len < tuple_len => Some(TupleMapping::new(0..columns_len, tuple_len)), None => None, } } diff --git a/tpcc/src/backend/mod.rs b/tpcc/src/backend/mod.rs index 4db98e18..e1ffdb48 100644 --- a/tpcc/src/backend/mod.rs +++ b/tpcc/src/backend/mod.rs @@ -19,12 +19,11 @@ pub mod kitesql_rocksdb; pub mod sqlite; use crate::TpccError; -use kite_sql::db::Program; +use kite_sql::db::{Param, Program}; use kite_sql::types::tuple::TupleImpl; -use kite_sql::types::value::DataValue; use kite_sql::types::LogicalType; -pub type DbParam = (usize, DataValue); +pub type DbParam<'a> = (usize, Param<'a>); pub trait SimpleExecutor { fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError>; diff --git a/tpcc/src/backend/sqlite.rs b/tpcc/src/backend/sqlite.rs index a1478159..576ab3e3 100644 --- a/tpcc/src/backend/sqlite.rs +++ b/tpcc/src/backend/sqlite.rs @@ -17,6 +17,7 @@ use super::{ }; use crate::TpccError; use chrono::{NaiveDateTime, TimeZone, Utc}; +use kite_sql::db::Param; use kite_sql::types::tuple::{Tuple, TupleImpl}; use kite_sql::types::value::{DataValue, Utf8Type}; use kite_sql::types::{CharLengthUnits, LogicalType}; @@ -216,8 +217,13 @@ impl<'a> BackendTransaction for SqliteTransaction<'a> { } fn bind_params(statement: &mut SqliteStatement<'_>, params: &[DbParam]) -> Result<(), TpccError> { - for (id, value) in params { - statement.bind((*id, convert_value(value)?))?; + for (id, param) in params { + let value = match param { + Param::Value(value) => convert_value(value)?, + Param::Str(value) => Value::String(value.to_string()), + Param::Bytes(value) => Value::Binary(value.to_vec()), + }; + statement.bind((*id, value))?; } Ok(()) } diff --git a/tpcc/src/delivery.rs b/tpcc/src/delivery.rs index 86e3a4a3..f73590ba 100644 --- a/tpcc/src/delivery.rs +++ b/tpcc/src/delivery.rs @@ -51,8 +51,8 @@ impl TpccTransaction for Delivery { tx.with_query_one( &mut statements[0], &[ - (1, DataValue::Int8(d_id as i8)), - (2, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Int8(d_id as i8).into()), + (2, DataValue::Int16(args.w_id as i16).into()), ], &mut |tuple| { no_o_id = tuple[0].i32().unwrap(); @@ -67,9 +67,9 @@ impl TpccTransaction for Delivery { tx.execute_drain( &mut statements[1], &[ - (1, DataValue::Int32(no_o_id)), - (2, DataValue::Int8(d_id as i8)), - (3, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Int32(no_o_id).into()), + (2, DataValue::Int8(d_id as i8).into()), + (3, DataValue::Int16(args.w_id as i16).into()), ], )?; // "SELECT o_c_id FROM orders WHERE o_id = ? AND o_d_id = ? AND o_w_id = ?" @@ -77,9 +77,9 @@ impl TpccTransaction for Delivery { tx.with_query_one( &mut statements[2], &[ - (1, DataValue::Int32(no_o_id)), - (2, DataValue::Int8(d_id as i8)), - (3, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Int32(no_o_id).into()), + (2, DataValue::Int8(d_id as i8).into()), + (3, DataValue::Int16(args.w_id as i16).into()), ], &mut |tuple| { c_id = tuple[0].i32().unwrap(); @@ -90,20 +90,20 @@ impl TpccTransaction for Delivery { tx.execute_drain( &mut statements[3], &[ - (1, DataValue::Int8(args.o_carrier_id as i8)), - (2, DataValue::Int32(no_o_id)), - (3, DataValue::Int8(d_id as i8)), - (4, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Int8(args.o_carrier_id as i8).into()), + (2, DataValue::Int32(no_o_id).into()), + (3, DataValue::Int8(d_id as i8).into()), + (4, DataValue::Int16(args.w_id as i16).into()), ], )?; // "UPDATE order_line SET ol_delivery_d = ? WHERE ol_o_id = ? AND ol_d_id = ? AND ol_w_id = ?" tx.execute_drain( &mut statements[4], &[ - (1, DataValue::from(&now)), - (2, DataValue::Int32(no_o_id)), - (3, DataValue::Int8(d_id as i8)), - (4, DataValue::Int16(args.w_id as i16)), + (1, DataValue::from(&now).into()), + (2, DataValue::Int32(no_o_id).into()), + (3, DataValue::Int8(d_id as i8).into()), + (4, DataValue::Int16(args.w_id as i16).into()), ], )?; // "SELECT SUM(ol_amount) FROM order_line WHERE ol_o_id = ? AND ol_d_id = ? AND ol_w_id = ?" @@ -111,9 +111,9 @@ impl TpccTransaction for Delivery { tx.with_query_one( &mut statements[5], &[ - (1, DataValue::Int32(no_o_id)), - (2, DataValue::Int8(d_id as i8)), - (3, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Int32(no_o_id).into()), + (2, DataValue::Int8(d_id as i8).into()), + (3, DataValue::Int16(args.w_id as i16).into()), ], &mut |tuple| { ol_total = tuple[0].decimal().unwrap(); @@ -124,10 +124,10 @@ impl TpccTransaction for Delivery { tx.execute_drain( &mut statements[6], &[ - (1, DataValue::Decimal(ol_total)), - (2, DataValue::Int32(c_id)), - (3, DataValue::Int8(d_id as i8)), - (4, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Decimal(ol_total).into()), + (2, DataValue::Int32(c_id).into()), + (3, DataValue::Int8(d_id as i8).into()), + (4, DataValue::Int16(args.w_id as i16).into()), ], )?; } diff --git a/tpcc/src/new_ord.rs b/tpcc/src/new_ord.rs index 7836c219..4ff00f25 100644 --- a/tpcc/src/new_ord.rs +++ b/tpcc/src/new_ord.rs @@ -88,10 +88,10 @@ impl TpccTransaction for NewOrd { tx.with_query_one( &mut statements[0], &[ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int16(args.w_id as i16)), - (3, DataValue::Int8(args.d_id as i8)), - (4, DataValue::Int64(args.c_id as i64)), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int16(args.w_id as i16).into()), + (3, DataValue::Int8(args.d_id as i8).into()), + (4, DataValue::Int64(args.c_id as i64).into()), ], &mut |tuple| { c_discount = tuple[0].decimal().unwrap(); @@ -111,9 +111,9 @@ impl TpccTransaction for NewOrd { tx.with_query_one( &mut statements[1], &[ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::Int32(args.c_id as i32)), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::Int32(args.c_id as i32).into()), ], &mut |tuple| { c_discount = tuple[0].decimal().unwrap(); @@ -126,7 +126,7 @@ impl TpccTransaction for NewOrd { let mut w_tax = Decimal::default(); tx.with_query_one( &mut statements[2], - &[(1, DataValue::Int16(args.w_id as i16))], + &[(1, DataValue::Int16(args.w_id as i16).into())], &mut |tuple| { w_tax = tuple[0].decimal().unwrap(); Ok(()) @@ -141,8 +141,8 @@ impl TpccTransaction for NewOrd { tx.with_query_one( &mut statements[3], &[ - (1, DataValue::Int8(args.d_id as i8)), - (2, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Int8(args.d_id as i8).into()), + (2, DataValue::Int16(args.w_id as i16).into()), ], &mut |tuple| { d_next_o_id = tuple[0].i32().unwrap(); @@ -154,9 +154,9 @@ impl TpccTransaction for NewOrd { tx.execute_drain( &mut statements[4], &[ - (1, DataValue::Int32(d_next_o_id)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Int32(d_next_o_id).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::Int16(args.w_id as i16).into()), ], )?; let o_id = d_next_o_id; @@ -164,22 +164,22 @@ impl TpccTransaction for NewOrd { tx.execute_drain( &mut statements[5], &[ - (1, DataValue::Int32(o_id)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::Int16(args.w_id as i16)), - (4, DataValue::Int32(args.c_id as i32)), - (5, DataValue::from(&now)), - (6, DataValue::Int8(args.o_ol_cnt as i8)), - (7, DataValue::Int8(args.o_all_local as i8)), + (1, DataValue::Int32(o_id).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::Int16(args.w_id as i16).into()), + (4, DataValue::Int32(args.c_id as i32).into()), + (5, DataValue::from(&now).into()), + (6, DataValue::Int8(args.o_ol_cnt as i8).into()), + (7, DataValue::Int8(args.o_all_local as i8).into()), ], )?; // "INSERT INTO new_orders (no_o_id, no_d_id, no_w_id) VALUES (?,?,?)" tx.execute_drain( &mut statements[6], &[ - (1, DataValue::Int32(o_id)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Int32(o_id).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::Int16(args.w_id as i16).into()), ], )?; let mut ol_num_seq = vec![0; MAX_NUM_ITEMS]; @@ -210,7 +210,7 @@ impl TpccTransaction for NewOrd { let ol_i_id = args.item_id[ol_num_seq[ol_number - 1]]; let ol_quantity = args.qty[ol_num_seq[ol_number - 1]]; // "SELECT i_price, i_name, i_data FROM item WHERE i_id = ?" - let params = [(1, DataValue::Int32(ol_i_id as i32))]; + let params = [(1, DataValue::Int32(ol_i_id as i32).into())]; let mut i_price = Decimal::default(); let mut i_name = String::new(); let mut i_data = String::new(); @@ -226,8 +226,8 @@ impl TpccTransaction for NewOrd { // "SELECT s_quantity, s_data, s_dist_01, s_dist_02, s_dist_03, s_dist_04, s_dist_05, s_dist_06, s_dist_07, s_dist_08, s_dist_09, s_dist_10 FROM stock WHERE s_i_id = ? AND s_w_id = ? FOR UPDATE" let params = [ - (1, DataValue::Int32(ol_i_id as i32)), - (2, DataValue::Int16(ol_supply_w_id as i16)), + (1, DataValue::Int32(ol_i_id as i32).into()), + (2, DataValue::Int16(ol_supply_w_id as i16).into()), ]; let mut s_quantity = 0; let mut s_data = String::new(); @@ -276,9 +276,9 @@ impl TpccTransaction for NewOrd { }; // "UPDATE stock SET s_quantity = ? WHERE s_i_id = ? AND s_w_id = ?" let params = [ - (1, DataValue::Int16(s_quantity)), - (2, DataValue::Int32(ol_i_id as i32)), - (3, DataValue::Int16(ol_supply_w_id as i16)), + (1, DataValue::Int16(s_quantity).into()), + (2, DataValue::Int32(ol_i_id as i32).into()), + (3, DataValue::Int16(ol_supply_w_id as i16).into()), ]; tx.execute_drain(&mut statements[9], ¶ms)?; @@ -294,15 +294,15 @@ impl TpccTransaction for NewOrd { amt[ol_num_seq[ol_number - 1]] = ol_amount; // "INSERT INTO order_line (ol_o_id, ol_d_id, ol_w_id, ol_number, ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_dist_info) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" let params = [ - (1, DataValue::Int32(o_id)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::Int16(args.w_id as i16)), - (4, DataValue::Int8(ol_number as i8)), - (5, DataValue::Int32(ol_i_id as i32)), - (6, DataValue::Int16(ol_supply_w_id as i16)), - (7, DataValue::Int8(ol_quantity as i8)), - (8, DataValue::Decimal(ol_amount.round_dp(2))), - (9, DataValue::from(ol_dist_info)), + (1, DataValue::Int32(o_id).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::Int16(args.w_id as i16).into()), + (4, DataValue::Int8(ol_number as i8).into()), + (5, DataValue::Int32(ol_i_id as i32).into()), + (6, DataValue::Int16(ol_supply_w_id as i16).into()), + (7, DataValue::Int8(ol_quantity as i8).into()), + (8, DataValue::Decimal(ol_amount.round_dp(2)).into()), + (9, DataValue::from(ol_dist_info).into()), ]; tx.execute_drain(&mut statements[10], ¶ms)?; } diff --git a/tpcc/src/order_stat.rs b/tpcc/src/order_stat.rs index 6bbd0dc4..f4fadd3e 100644 --- a/tpcc/src/order_stat.rs +++ b/tpcc/src/order_stat.rs @@ -63,9 +63,9 @@ impl TpccTransaction for OrderStat { tx.with_query_one( &mut statements[0], &[ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::from(args.c_last.clone())), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::from(args.c_last.clone()).into()), ], &mut |tuple| { name_cnt = tuple[0].i32().unwrap() as usize; @@ -74,9 +74,9 @@ impl TpccTransaction for OrderStat { )?; // "SELECT c_balance, c_first, c_middle, c_last FROM customer WHERE c_w_id = ? AND c_d_id = ? AND c_last = ? ORDER BY c_first" let params = [ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::from(args.c_last.clone())), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::from(args.c_last.clone()).into()), ]; if name_cnt % 2 == 1 { name_cnt += 1; @@ -103,9 +103,9 @@ impl TpccTransaction for OrderStat { tx.with_query_one( &mut statements[2], &[ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::Int32(args.c_id as i32)), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::Int32(args.c_id as i32).into()), ], &mut |tuple| { c_balance = tuple[0].decimal().unwrap(); @@ -119,12 +119,12 @@ impl TpccTransaction for OrderStat { }; // "SELECT o_id, o_entry_d, COALESCE(o_carrier_id,0) FROM orders WHERE o_w_id = ? AND o_d_id = ? AND o_c_id = ? AND o_id = (SELECT MAX(o_id) FROM orders WHERE o_w_id = ? AND o_d_id = ? AND o_c_id = ?)" let params = [ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::Int32(args.c_id as i32)), - (4, DataValue::Int16(args.w_id as i16)), - (5, DataValue::Int8(args.d_id as i8)), - (6, DataValue::Int32(args.c_id as i32)), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::Int32(args.c_id as i32).into()), + (4, DataValue::Int16(args.w_id as i16).into()), + (5, DataValue::Int8(args.d_id as i8).into()), + (6, DataValue::Int32(args.c_id as i32).into()), ]; let mut o_id = 0; tx.with_query_one(&mut statements[3], ¶ms, &mut |tuple| { @@ -133,9 +133,9 @@ impl TpccTransaction for OrderStat { })?; // "SELECT ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_delivery_d FROM order_line WHERE ol_w_id = ? AND ol_d_id = ? AND ol_o_id = ?" let params = [ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::Int32(o_id)), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::Int32(o_id).into()), ]; tx.with_query_one(&mut statements[4], ¶ms, &mut |_| Ok(()))?; // let ol_i_id = tuple[0].i32(); diff --git a/tpcc/src/payment.rs b/tpcc/src/payment.rs index 27eed290..d07a4cee 100644 --- a/tpcc/src/payment.rs +++ b/tpcc/src/payment.rs @@ -75,8 +75,8 @@ impl TpccTransaction for Payment { tx.execute_drain( &mut statements[0], &[ - (1, DataValue::Decimal(args.h_amount)), - (2, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Decimal(args.h_amount).into()), + (2, DataValue::Int16(args.w_id as i16).into()), ], )?; // "SELECT w_street_1, w_street_2, w_city, w_state, w_zip, w_name FROM warehouse WHERE w_id = ?" @@ -88,7 +88,7 @@ impl TpccTransaction for Payment { let mut w_name = String::new(); tx.with_query_one( &mut statements[1], - &[(1, DataValue::Int16(args.w_id as i16))], + &[(1, DataValue::Int16(args.w_id as i16).into())], &mut |tuple| { w_street_1 = tuple[0].utf8().unwrap().to_string(); w_street_2 = tuple[1].utf8().unwrap().to_string(); @@ -104,9 +104,9 @@ impl TpccTransaction for Payment { tx.execute_drain( &mut statements[2], &[ - (1, DataValue::Decimal(args.h_amount)), - (2, DataValue::Int16(args.w_id as i16)), - (3, DataValue::Int8(args.d_id as i8)), + (1, DataValue::Decimal(args.h_amount).into()), + (2, DataValue::Int16(args.w_id as i16).into()), + (3, DataValue::Int8(args.d_id as i8).into()), ], )?; @@ -120,8 +120,8 @@ impl TpccTransaction for Payment { tx.with_query_one( &mut statements[3], &[ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int8(args.d_id as i8)), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int8(args.d_id as i8).into()), ], &mut |tuple| { d_street_1 = tuple[0].utf8().unwrap().to_string(); @@ -141,9 +141,9 @@ impl TpccTransaction for Payment { tx.with_query_one( &mut statements[4], &[ - (1, DataValue::Int16(args.c_w_id as i16)), - (2, DataValue::Int8(args.c_d_id as i8)), - (3, DataValue::from(args.c_last.clone())), + (1, DataValue::Int16(args.c_w_id as i16).into()), + (2, DataValue::Int8(args.c_d_id as i8).into()), + (3, DataValue::from(args.c_last.clone()).into()), ], &mut |tuple| { name_cnt = tuple[0].i32().unwrap(); @@ -152,9 +152,9 @@ impl TpccTransaction for Payment { )?; // "SELECT c_id FROM customer WHERE c_w_id = ? AND c_d_id = ? AND c_last = ? ORDER BY c_first" let params = [ - (1, DataValue::Int16(args.c_w_id as i16)), - (2, DataValue::Int8(args.c_d_id as i8)), - (3, DataValue::from(args.c_last.clone())), + (1, DataValue::Int16(args.c_w_id as i16).into()), + (2, DataValue::Int8(args.c_d_id as i8).into()), + (3, DataValue::from(args.c_last.clone()).into()), ]; if name_cnt % 2 == 1 { name_cnt += 1; @@ -183,9 +183,9 @@ impl TpccTransaction for Payment { tx.with_query_one( &mut statements[6], &[ - (1, DataValue::Int16(args.c_w_id as i16)), - (2, DataValue::Int8(args.c_d_id as i8)), - (3, DataValue::Int32(c_id)), + (1, DataValue::Int16(args.c_w_id as i16).into()), + (2, DataValue::Int8(args.c_d_id as i8).into()), + (3, DataValue::Int32(c_id).into()), ], &mut |tuple| { c_first = tuple[0].utf8().unwrap().to_string(); @@ -214,9 +214,9 @@ impl TpccTransaction for Payment { tx.with_query_one( &mut statements[7], &[ - (1, DataValue::Int16(args.c_w_id as i16)), - (2, DataValue::Int8(args.c_d_id as i8)), - (3, DataValue::Int32(c_id)), + (1, DataValue::Int16(args.c_w_id as i16).into()), + (2, DataValue::Int8(args.c_d_id as i8).into()), + (3, DataValue::Int32(c_id).into()), ], &mut |tuple| { c_data = tuple[0].utf8().unwrap().to_string(); @@ -231,11 +231,11 @@ impl TpccTransaction for Payment { tx.execute_drain( &mut statements[8], &[ - (1, DataValue::Decimal(c_balance)), - (2, DataValue::from(c_data)), - (3, DataValue::Int16(args.c_w_id as i16)), - (4, DataValue::Int8(args.c_d_id as i8)), - (5, DataValue::Int32(c_id)), + (1, DataValue::Decimal(c_balance).into()), + (2, DataValue::from(c_data).into()), + (3, DataValue::Int16(args.c_w_id as i16).into()), + (4, DataValue::Int8(args.c_d_id as i8).into()), + (5, DataValue::Int32(c_id).into()), ], )?; } else { @@ -243,10 +243,10 @@ impl TpccTransaction for Payment { tx.execute_drain( &mut statements[9], &[ - (1, DataValue::Decimal(c_balance)), - (2, DataValue::Int16(args.c_w_id as i16)), - (3, DataValue::Int8(args.c_d_id as i8)), - (4, DataValue::Int32(c_id)), + (1, DataValue::Decimal(c_balance).into()), + (2, DataValue::Int16(args.c_w_id as i16).into()), + (3, DataValue::Int8(args.c_d_id as i8).into()), + (4, DataValue::Int32(c_id).into()), ], )?; } @@ -255,10 +255,10 @@ impl TpccTransaction for Payment { tx.execute_drain( &mut statements[9], &[ - (1, DataValue::Decimal(c_balance)), - (2, DataValue::Int16(args.c_w_id as i16)), - (3, DataValue::Int8(args.c_d_id as i8)), - (4, DataValue::Int32(c_id)), + (1, DataValue::Decimal(c_balance).into()), + (2, DataValue::Int16(args.c_w_id as i16).into()), + (3, DataValue::Int8(args.c_d_id as i8).into()), + (4, DataValue::Int32(c_id).into()), ], )?; } @@ -267,14 +267,17 @@ impl TpccTransaction for Payment { tx.execute_drain( &mut statements[10], &[ - (1, DataValue::Int8(args.c_d_id as i8)), - (2, DataValue::Int16(args.c_w_id as i16)), - (3, DataValue::Int32(c_id)), - (4, DataValue::Int8(args.d_id as i8)), - (5, DataValue::Int16(args.w_id as i16)), - (6, DataValue::Time64(now.timestamp_micros(), 6, false)), - (7, DataValue::Decimal(args.h_amount)), - (8, DataValue::from(h_data)), + (1, DataValue::Int8(args.c_d_id as i8).into()), + (2, DataValue::Int16(args.c_w_id as i16).into()), + (3, DataValue::Int32(c_id).into()), + (4, DataValue::Int8(args.d_id as i8).into()), + (5, DataValue::Int16(args.w_id as i16).into()), + ( + 6, + DataValue::Time64(now.timestamp_micros(), 6, false).into(), + ), + (7, DataValue::Decimal(args.h_amount).into()), + (8, DataValue::from(h_data).into()), ], )?; diff --git a/tpcc/src/slev.rs b/tpcc/src/slev.rs index 6c54f7de..50b6da3a 100644 --- a/tpcc/src/slev.rs +++ b/tpcc/src/slev.rs @@ -48,8 +48,8 @@ impl TpccTransaction for Slev { tx.with_query_one( &mut statements[0], &[ - (1, DataValue::Int8(args.d_id as i8)), - (2, DataValue::Int16(args.w_id as i16)), + (1, DataValue::Int8(args.d_id as i8).into()), + (2, DataValue::Int16(args.w_id as i16).into()), ], &mut |tuple| { d_next_o_id = tuple[0].i32().unwrap(); @@ -61,10 +61,10 @@ impl TpccTransaction for Slev { tx.with_query_all( &mut statements[1], &[ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int8(args.d_id as i8)), - (3, DataValue::Int32(d_next_o_id)), - (4, DataValue::Int32(d_next_o_id)), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int8(args.d_id as i8).into()), + (3, DataValue::Int32(d_next_o_id).into()), + (4, DataValue::Int32(d_next_o_id).into()), ], &mut |tuple| { item_ids.push(tuple[0].i32().unwrap()); @@ -77,9 +77,9 @@ impl TpccTransaction for Slev { tx.with_query_one( &mut statements[2], &[ - (1, DataValue::Int16(args.w_id as i16)), - (2, DataValue::Int32(item_id)), - (3, DataValue::Int16(args.level as i16)), + (1, DataValue::Int16(args.w_id as i16).into()), + (2, DataValue::Int32(item_id).into()), + (3, DataValue::Int16(args.level as i16).into()), ], &mut |tuple| { _low_stock += tuple[0].i32().unwrap(); From cbe0aae4407aab1a42ad4aec5ca50a3a50b0e2c6 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 4 Nov 2026 18:45:34 +0800 Subject: [PATCH 06/14] perf(scan): skip unread row columns and move plain projected columns - Fixed-size integers decode straight from the row slice. - SkipFixed and SkipVariable carry `null`: with it the column's bytes are skipped but it keeps its slot as a NULL. SkipResidualFilterColumns uses it for table-row reads (looked-up index rows, primary key and sequential scans), so a column only the eliminated index predicate read is no longer decoded. Covered reads keep using the cover mapping. - ProjectOperator::exprs holds `(Option, ExprRef)`. The last optimizer rule, MapProjectColumns, sets the input position of each plain column no other projected expression reads, and rewrite moves that value out instead of evaluating and cloning it. - ProjectOperator::new takes an iterator of expressions. --- src/binder/parser.rs | 3 +- src/binder/select.rs | 28 ++--- src/db.rs | 5 + src/execution/dql/projection.rs | 2 +- src/execution/mod.rs | 26 +++++ src/optimizer/plan_utils.rs | 2 +- .../rule/normalization/column_pruning.rs | 88 +++++++++++---- .../rule/normalization/elimination.rs | 98 ++++++++++++----- .../rule/normalization/guard_mark_apply.rs | 2 +- .../rule/normalization/join_reorder.rs | 24 ++--- src/optimizer/rule/normalization/mod.rs | 10 +- .../rule/normalization/parameterized_index.rs | 13 +-- .../rule/normalization/simplification.rs | 8 +- src/planner/mod.rs | 2 +- src/planner/operator/mod.rs | 2 +- src/planner/operator/project.rs | 10 +- src/planner/operator/visitor.rs | 2 +- src/planner/operator/visitor_mut.rs | 2 +- src/storage/mod.rs | 31 +++++- src/types/serialize.rs | 102 ++++++++++++------ src/types/tuple.rs | 8 +- 21 files changed, 334 insertions(+), 134 deletions(-) diff --git a/src/binder/parser.rs b/src/binder/parser.rs index 9aac5ad4..e63b74b5 100644 --- a/src/binder/parser.rs +++ b/src/binder/parser.rs @@ -1291,8 +1291,7 @@ where column, target_offset + index, )) - }) - .collect(); + }); plan = LogicalPlan::new( Operator::Project(ProjectOperator::new(exprs)), Childrens::Only(Box::new(plan)), diff --git a/src/binder/select.rs b/src/binder/select.rs index 31ef70ea..9f4e23ce 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -77,11 +77,11 @@ struct AppendedRightOutput { } struct ProjectionOutputBinder<'a> { - project_exprs: &'a [ExprRef], + project_exprs: &'a [(Option, ExprRef)], } impl<'a> ProjectionOutputBinder<'a> { - fn new(project_exprs: &'a [ExprRef]) -> Self { + fn new(project_exprs: &'a [(Option, ExprRef)]) -> Self { Self { project_exprs } } @@ -92,14 +92,14 @@ impl<'a> ProjectionOutputBinder<'a> { ) -> Option { self.project_exprs .iter() - .position(|candidate| { + .position(|(_, candidate)| { candidate.eq_ignore_colref_pos(expr, arena) || candidate .unpack_alias(arena) .eq_ignore_colref_pos(expr.unpack_alias(arena), arena) }) .map(|position| { - let output_expr = self.project_exprs[position]; + let output_expr = self.project_exprs[position].1; ScalarExpression::column_expr(output_expr.output_column_ref(arena), position) }) } @@ -864,9 +864,13 @@ impl<'a: 'b, 'b, T: StorageTransaction, A: AsRef<[(usize, LogicalType)]>> Binder /// Whether `exprs` only renames its input to an alias, as `bind_alias` builds for `FROM t AS x` /// (`temp == false`) or for a temp table (`temp == true`). - fn is_alias_projection(exprs: &[ExprRef], temp: bool, arena: &PlanArena) -> bool { + fn is_alias_projection( + exprs: &[(Option, ExprRef)], + temp: bool, + arena: &PlanArena, + ) -> bool { !exprs.is_empty() - && exprs.iter().all(|expr| { + && exprs.iter().all(|(_, expr)| { matches!( arena.expression(*expr), ScalarExpression::Alias { @@ -1660,9 +1664,10 @@ impl<'a: 'b, 'b, T: StorageTransaction, A: AsRef<[(usize, LogicalType)]>> Binder } } if referenced { - op.exprs.push( + op.exprs.push(( + None, arena.alloc_expression(ScalarExpression::column_expr(*column, position)), - ); + )); appended_outputs.push(AppendedRightOutput { column: *column, child_position: position, @@ -1864,7 +1869,7 @@ impl<'a: 'b, 'b, T: StorageTransaction, A: AsRef<[(usize, LogicalType)]>> Binder if !preserve_projection || Self::is_alias_projection(&op.exprs, true, arena) { Ok((child, correlated_filters)) } else { - for expr in &op.exprs { + for (_, expr) in &op.exprs { if Self::expr_has_correlated_refs(*expr, left_schema, arena)? { return Err(DatabaseError::UnsupportedStmt( "correlated references in the SELECT list of an IN/ANY/ALL subquery are not supported" @@ -2313,8 +2318,7 @@ mod tests { alias: AliasType::Name("v".to_string()), }); - ProjectionOutputBinder::new(std::slice::from_ref(&project_output)) - .visit(&mut expr, &mut arena)?; + ProjectionOutputBinder::new(&[(None, project_output)]).visit(&mut expr, &mut arena)?; let output_column = project_output.output_column_ref(&mut arena); let expected = arena.alloc_expression(ScalarExpression::column_expr(output_column, 0)); @@ -2408,7 +2412,7 @@ mod tests { let Operator::Project(project) = &plan.operator else { unreachable!() }; - let selected = project.exprs[1].unpack_alias(&arena); + let selected = project.exprs[1].1.unpack_alias(&arena); assert!( matches!(arena.expression(selected), ScalarExpression::ColumnRef { column, position: 2 } if arena.same_column(*column, marker)) diff --git a/src/db.rs b/src/db.rs index f0ab6b51..3f3028fa 100644 --- a/src/db.rs +++ b/src/db.rs @@ -550,6 +550,11 @@ fn optimizer_pipeline() -> HepOptimizerPipeline { HepBatchStrategy::once_topdown(), &[NormalizationRuleImpl::SkipResidualFilterColumns], ) + .after_batch( + "Map Project Columns", + HepBatchStrategy::fix_point_topdown(1), + &[NormalizationRuleImpl::MapProjectColumns], + ) .implementations(&[ // DQL ImplementationRuleImpl::SimpleAggregate, diff --git a/src/execution/dql/projection.rs b/src/execution/dql/projection.rs index 81dd6c57..a73b72ac 100644 --- a/src/execution/dql/projection.rs +++ b/src/execution/dql/projection.rs @@ -24,7 +24,7 @@ use crate::storage::StorageTransaction; use crate::types::value::DataValue; pub struct Projection<'a> { - exprs: &'a [ExprRef], + exprs: &'a [(Option, ExprRef)], input: ExecNode<'a>, } diff --git a/src/execution/mod.rs b/src/execution/mod.rs index 38e74505..5d31878e 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -115,18 +115,39 @@ impl ExecResult { /// Resolves either an arena-backed expression or a direct scalar expression. pub(crate) trait RewriteExpression { fn expression<'a>(&'a self, arena: &'a dyn MetaArena) -> &'a ScalarExpression; + + /// The position of the rewritten row this expression moves out instead of evaluating. + fn moved(&self) -> Option; } impl RewriteExpression for ExprRef { fn expression<'a>(&'a self, arena: &'a dyn MetaArena) -> &'a ScalarExpression { arena.expression(*self) } + + fn moved(&self) -> Option { + None + } } impl RewriteExpression for ScalarExpression { fn expression<'a>(&'a self, _arena: &'a dyn MetaArena) -> &'a ScalarExpression { self } + + fn moved(&self) -> Option { + None + } +} + +impl RewriteExpression for (Option, ExprRef) { + fn expression<'a>(&'a self, arena: &'a dyn MetaArena) -> &'a ScalarExpression { + arena.expression(self.1) + } + + fn moved(&self) -> Option { + self.0 + } } /// A statement built into an executor tree, like a compiled SQLite program. @@ -943,6 +964,11 @@ impl<'c, 'a, 't, T: StorageTransaction + 'a> ExecContext<'c, 'a, 't, T> { let pool = self.buffer.table_codec.bytes_pool(); result.spare_values.reserve(exprs.len()); for expr in exprs { + if let (Some(position), None) = (expr.moved(), input) { + let value = mem::replace(&mut result.tuple[position], DataValue::Null); + result.spare_values.push(value); + continue; + } let expression = expr.expression(arena); let value = match input { Some(input) => expression.eval(arena, Some(input)), diff --git a/src/optimizer/plan_utils.rs b/src/optimizer/plan_utils.rs index d5bc0a32..7a24554c 100644 --- a/src/optimizer/plan_utils.rs +++ b/src/optimizer/plan_utils.rs @@ -205,7 +205,7 @@ impl<'plan, A: MetaArena> OperatorVisitor<'plan> for ScanColumnVisitor<'_, A> { return Ok(()); }; let arena = self.arena; - for &candidate in &project.exprs { + for &(_, candidate) in &project.exprs { let source = match arena.expression(candidate) { ScalarExpression::Alias { expr, diff --git a/src/optimizer/rule/normalization/column_pruning.rs b/src/optimizer/rule/normalization/column_pruning.rs index 37d7d148..b85c21b6 100644 --- a/src/optimizer/rule/normalization/column_pruning.rs +++ b/src/optimizer/rule/normalization/column_pruning.rs @@ -237,9 +237,10 @@ impl ColumnPruning { column_references.contains(output_column, arena) } - fn clear_exprs( + fn clear_exprs( column_references: &ReferencedColumns, - exprs: &mut Vec, + exprs: &mut Vec, + expr_of: fn(&E) -> ExprRef, removed_positions: &mut Vec, output_start: usize, arena: &mut crate::planner::PlanArena, @@ -248,7 +249,7 @@ impl ColumnPruning { removed_positions.reserve(exprs.len()); let mut position = 0; exprs.retain(|expr| { - let keep = Self::output_column_is_required(*expr, column_references, arena); + let keep = Self::output_column_is_required(expr_of(expr), column_references, arena); if !keep { removed_positions.push(position); } @@ -366,6 +367,7 @@ impl ColumnPruning { Self::clear_exprs( &required_columns, &mut op.agg_calls, + |expr| *expr, &mut outcome.removed_positions, output_start, arena, @@ -429,7 +431,7 @@ impl ColumnPruning { } Operator::Project(op) => { let mut has_count_star = HasCountStar::default(); - for expr in &op.exprs { + for (_, expr) in &op.exprs { has_count_star.visit(*expr, arena)?; } if !has_count_star.value { @@ -437,6 +439,7 @@ impl ColumnPruning { Self::clear_exprs( &required_columns, &mut op.exprs, + |(_, expr)| *expr, &mut outcome.removed_positions, output_start, arena, @@ -452,7 +455,7 @@ impl ColumnPruning { let child_changed = { required_columns.clear(); Self::extend_expr_referenced_columns( - op.exprs.iter(), + op.exprs.iter().map(|(_, expr)| expr), &mut required_columns, arena, )?; @@ -859,20 +862,26 @@ impl SkipResidualFilterColumns { changed |= Self::skip(right, referenced, arena); } } - let ( - Operator::TableScan(scan), + let Operator::TableScan(scan) = &mut plan.operator else { + return changed; + }; + let info = match plan.physical_option.as_mut() { + // A primary key read decodes the table row rather than index fields. Some(PhysicalOption { plan: PlanImpl::IndexScan(info), .. - }), - ) = (&plan.operator, plan.physical_option.as_mut()) - else { - return changed; + }) if info.covered + && !matches!(arena.index(info.meta).ty, IndexType::PrimaryKey { .. }) => + { + info + } + _ => { + return changed + | scan + .deserializers + .null_unread(arena, |column| referenced.contains(column, arena)) + } }; - // A primary key read decodes the table row rather than index fields. - if !info.covered || matches!(arena.index(info.meta).ty, IndexType::PrimaryKey { .. }) { - return changed; - } let len = scan.columns().count(); for (position, column) in scan.deserializers.read_columns().enumerate() { let skipped = info @@ -1175,9 +1184,10 @@ mod tests { } #[test] - fn skip_residual_filter_columns_in_covered_index_reads() -> Result<(), DatabaseError> { + fn skip_residual_filter_columns_in_index_reads() -> Result<(), DatabaseError> { use crate::db::DataBaseBuilder; use crate::planner::operator::{PhysicalOption, PlanImpl}; + use crate::types::serialize::TupleValueSerializableImpl; use crate::types::tuple::TupleLike; use crate::types::value::DataValue; @@ -1190,10 +1200,15 @@ mod tests { let mut db = DataBaseBuilder::path(".") .histogram_buckets(2) .build_in_memory()?; - db.ddl("create table skip_t(id int primary key, a int, b int)")?; + db.ddl("create table skip_t(id int primary key, a int, b int, c int)")?; db.ddl("create index skip_ab on skip_t(a, b)")?; - db.run("insert into skip_t values(1, 1, 10), (2, 1, 20), (3, 2, 30)")? + db.run("insert into skip_t values(1, 1, 10, 100), (2, 1, 20, 200), (3, 2, 30, 300)")? .done()?; + // Rows of other `a`s, so that `a = 1` is selective enough for the index. + for id in 4..64 { + db.run(format!("insert into skip_t values({id}, {id}, 0, 0)"))? + .done()?; + } db.analyze("skip_t")?; let mut tx = db.new_transaction(false)?; @@ -1242,10 +1257,45 @@ mod tests { assert_eq!(rows, expected, "{sql}"); } + // A row looked up through the index: the range eliminates `a = 1`, so `a` keeps its slot + // as a NULL without being decoded. + let mut program = db.program_sql("select c from skip_t where a = 1 and c > 150", &[])?; + let scan = find_scan(program.executor.plan()).expect("table scan"); + let ( + Operator::TableScan(op), + Some(PhysicalOption { + plan: PlanImpl::IndexScan(info), + .. + }), + ) = (&scan.operator, &scan.physical_option) + else { + panic!("expected an index scan") + }; + assert!(!info.covered); + let skipped = op + .deserializers + .read_columns() + .map(|column| { + matches!( + column.deserializer, + TupleValueSerializableImpl::SkipFixed { null: true, .. } + ) + }) + .collect::>(); + assert_eq!(skipped, vec![true, false]); + let mut iter = program.execute(&mut tx, [])?; + let mut rows = Vec::new(); + while iter + .next_tuple(|_, row| rows.push(row.value_at(0).clone()))? + .is_some() + {} + iter.done()?; + assert_eq!(rows, vec![DataValue::Int32(200)]); + // An update writes back the whole row it scans, so its scan stays pinned. drop(tx); db.run("update skip_t set b = b + 1 where a = 1")?.done()?; - let mut iter = db.run("select id, a, b from skip_t order by id")?; + let mut iter = db.run("select id, a, b from skip_t where id <= 3 order by id")?; let mut rows = Vec::new(); while iter .next_tuple(|_, row| { diff --git a/src/optimizer/rule/normalization/elimination.rs b/src/optimizer/rule/normalization/elimination.rs index 232b4ef7..d18263ff 100644 --- a/src/optimizer/rule/normalization/elimination.rs +++ b/src/optimizer/rule/normalization/elimination.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::catalog::ColumnRef; use crate::errors::DatabaseError; use crate::expression::visitor::ExprVisitor; use crate::expression::visitor_mut::ExprVisitorMut; @@ -36,7 +37,7 @@ use crate::types::index::{IndexLookup, IndexOrderHint}; pub struct EliminateProject; struct Substitute<'m> { - exprs: Option<&'m [ExprRef]>, + exprs: Option<&'m [(Option, ExprRef)]>, offset: usize, } @@ -80,7 +81,7 @@ impl ExprVisitorMut for Substitute<'_> { match arena.expression(*expr) { ScalarExpression::ColumnRef { position, column } => { let (position, column) = (*position, *column); - if let Some(replacement) = self.exprs.and_then(|exprs| exprs.get(position)) { + if let Some((_, replacement)) = self.exprs.and_then(|exprs| exprs.get(position)) { // Do not revisit or mutate the replacement's input expression. *expr = *replacement; } else if self.offset != 0 { @@ -113,7 +114,10 @@ fn movable_project(plan: &LogicalPlan, arena: &PlanArena) -> Result>(), ) } else { @@ -239,7 +244,7 @@ impl NormalizationRule for EliminateProject { match &mut plan.operator { Operator::Project(op) => { if let Some(schema) = schema { - for (expr, column) in op.exprs.iter_mut().zip(schema) { + for ((_, expr), column) in op.exprs.iter_mut().zip(schema) { let current = expr.output_column_ref(arena); if arena.column(current).name() != arena.column(column).name() { let alias = AliasType::Name(arena.column(column).name().to_string()); @@ -275,6 +280,52 @@ impl NormalizationRule for EliminateProject { } } +pub struct MapProjectColumns; + +struct ColumnPositions(Vec); + +impl ExprVisitor for ColumnPositions { + fn visit_column_ref(&mut self, _: &ColumnRef, position: usize) -> Result<(), DatabaseError> { + self.0.push(position); + Ok(()) + } +} + +impl NormalizationRule for MapProjectColumns { + fn apply( + &self, + plan: &mut LogicalPlan, + arena: &mut PlanArena, + _loader: Option<&StatisticMetaLoader<'_>>, + ) -> Result { + let Operator::Project(op) = &mut plan.operator else { + return Ok(false); + }; + let mut reads = ColumnPositions(Vec::new()); + for (_, expr) in &op.exprs { + reads.visit(*expr, &*arena)?; + } + let mut changed = false; + for (moved, expr) in op.exprs.iter_mut().filter(|(moved, _)| moved.is_none()) { + let mut expression = arena.expression(*expr); + let position = loop { + match expression { + ScalarExpression::ColumnRef { position, .. } => break Some(*position), + ScalarExpression::Alias { expr, .. } => expression = arena.expression(*expr), + _ => break None, + } + }; + if let Some(position) = position + .filter(|position| reads.0.iter().filter(|read| *read == position).count() == 1) + { + *moved = Some(position); + changed = true; + } + } + Ok(changed) + } +} + pub struct EliminateRedundantSort; impl NormalizationRule for EliminateRedundantSort { @@ -850,10 +901,10 @@ mod tests { let Operator::Project(op) = plan.operator else { unreachable!() }; - let expr = binary(arena, op.exprs[1], BinaryOperator::Plus, 1); + let expr = binary(arena, op.exprs[1].1, BinaryOperator::Plus, 1); let input = plan.childrens.pop_only(); plan = LogicalPlan::new( - Operator::Project(ProjectOperator::new(vec![expr, op.exprs[0]])), + Operator::Project(ProjectOperator::new(vec![expr, op.exprs[0].1])), Childrens::Only(Box::new(input)), ); (plan, expr) @@ -913,7 +964,7 @@ mod tests { let Operator::Project(op) = &plan.operator else { unreachable!() }; - let ScalarExpression::Binary { left_expr, .. } = op.exprs[0].unpack_alias_ref(&arena) + let ScalarExpression::Binary { left_expr, .. } = op.exprs[0].1.unpack_alias_ref(&arena) else { unreachable!() }; @@ -977,7 +1028,7 @@ mod tests { let Operator::Project(op) = &plan.operator else { unreachable!() }; - assert_eq!(op.exprs[0], output); + assert_eq!(op.exprs[0].1, output); let Childrens::Only(aggregate) = plan.childrens.as_ref() else { unreachable!() }; @@ -994,7 +1045,7 @@ mod tests { let Operator::Project(op) = &input.operator else { unreachable!() }; - assert_eq!(op.exprs[0], replacement); + assert_eq!(op.exprs[0].1, replacement); assert!(matches!( input.childrens.only().operator, Operator::TableScan(_) @@ -1031,7 +1082,7 @@ mod tests { let Operator::Filter(filter) = &plan.childrens.only().operator else { unreachable!() }; - for rewritten in [project.exprs[0], filter.predicate] { + for rewritten in [project.exprs[0].1, filter.predicate] { let ScalarExpression::Binary { left_expr, .. } = rewritten.unpack_alias_ref(&arena) else { unreachable!() @@ -1068,12 +1119,9 @@ mod tests { fn permutation(arena: &mut PlanArena, mut input: LogicalPlan, slots: &[usize]) -> LogicalPlan { let schema = input.output_schema(arena).clone(); - let exprs = slots - .iter() - .map(|position| { - arena.alloc_expression(ScalarExpression::column_expr(schema[*position], *position)) - }) - .collect(); + let exprs = slots.iter().map(|position| { + arena.alloc_expression(ScalarExpression::column_expr(schema[*position], *position)) + }); LogicalPlan::new( Operator::Project(ProjectOperator::new(exprs)), Childrens::Only(Box::new(input)), @@ -1100,7 +1148,7 @@ mod tests { let Operator::Project(op) = &mut plan.operator else { unreachable!() }; - for expr in &mut op.exprs { + for (_, expr) in &mut op.exprs { *expr = binary(&mut arena, *expr, BinaryOperator::Multiply, 2); } let schema = plan.output_schema(&mut arena).clone(); @@ -1111,7 +1159,7 @@ mod tests { let Operator::Project(op) = &plan.operator else { unreachable!() }; - for (expr, expected) in op.exprs.iter().zip([0, 1, 2, 10, 11, 8, 9, 3, 4, 5, 6, 7]) { + for ((_, expr), expected) in op.exprs.iter().zip([0, 1, 2, 10, 11, 8, 9, 3, 4, 5, 6, 7]) { let ScalarExpression::Binary { left_expr, .. } = expr.unpack_alias_ref(&arena) else { unreachable!() }; @@ -1169,7 +1217,7 @@ mod tests { let positions: Vec<_> = op .exprs .iter() - .map(|expr| match expr.unpack_alias_ref(&arena) { + .map(|(_, expr)| match expr.unpack_alias_ref(&arena) { ScalarExpression::ColumnRef { position, .. } => *position, _ => unreachable!(), }) @@ -1196,7 +1244,7 @@ mod tests { let Operator::Project(op) = &plan.operator else { unreachable!() }; - for (expr, expected) in [(op.exprs[1], 2), (original, 1)] { + for (expr, expected) in [(op.exprs[1].1, 2), (original, 1)] { let ScalarExpression::Binary { left_expr, .. } = expr.unpack_alias_ref(&arena) else { unreachable!() }; @@ -1259,7 +1307,7 @@ mod tests { let Operator::Project(op) = &plan.operator else { unreachable!("expected project"); }; - let ScalarExpression::ColumnRef { position, .. } = op.exprs[0].unpack_alias_ref(&arena) + let ScalarExpression::ColumnRef { position, .. } = op.exprs[0].1.unpack_alias_ref(&arena) else { unreachable!("expected column ref"); }; diff --git a/src/optimizer/rule/normalization/guard_mark_apply.rs b/src/optimizer/rule/normalization/guard_mark_apply.rs index d53365e3..65bb0c3b 100644 --- a/src/optimizer/rule/normalization/guard_mark_apply.rs +++ b/src/optimizer/rule/normalization/guard_mark_apply.rs @@ -695,7 +695,7 @@ mod tests { }; project .exprs - .push(if escaped == 1 { fixture.marker } else { upper }); + .push((None, if escaped == 1 { fixture.marker } else { upper })); } assert_eq!( GuardMarkApply.apply(&mut fixture.plan, &mut arena, None)?, diff --git a/src/optimizer/rule/normalization/join_reorder.rs b/src/optimizer/rule/normalization/join_reorder.rs index 756c4260..fa0d8ef2 100644 --- a/src/optimizer/rule/normalization/join_reorder.rs +++ b/src/optimizer/rule/normalization/join_reorder.rs @@ -64,8 +64,7 @@ fn project_join_output( .zip(positions) .map(|(column, position)| { arena.alloc_expression(ScalarExpression::column_expr(column, position)) - }) - .collect(); + }); let mut plan = LogicalPlan::new( Operator::Project(ProjectOperator::new(exprs)), Childrens::Only(Box::new(join)), @@ -1126,17 +1125,12 @@ mod tests { }, )); let schema = join.output_schema(arena); - let exprs = schema - .iter() - .enumerate() - .map(|(position, column)| { - arena.alloc_expression(ScalarExpression::column_expr(*column, position)) - }) - .collect(); - let mut project = LogicalPlan::new( - Operator::Project(ProjectOperator::new(exprs)), - Childrens::Only(Box::new(join)), - ); + let exprs = schema.iter().enumerate().map(|(position, column)| { + arena.alloc_expression(ScalarExpression::column_expr(*column, position)) + }); + let project = ProjectOperator::new(exprs); + let mut project = + LogicalPlan::new(Operator::Project(project), Childrens::Only(Box::new(join))); project.physical_option = Some(PhysicalOption::new( PlanImpl::Project, SortOption::Follow, @@ -1210,7 +1204,7 @@ mod tests { let Operator::Project(project) = &plan.operator else { unreachable!() }; - for (expr, position) in project.exprs.iter().zip([2, 0, 1]) { + for ((_, expr), position) in project.exprs.iter().zip([2, 0, 1]) { assert!(matches!( arena.expression(*expr), ScalarExpression::ColumnRef { position: actual, .. } if *actual == position @@ -1318,7 +1312,7 @@ mod tests { let Operator::Project(project) = &plan.operator else { unreachable!() }; - for (expr, expected) in project.exprs.iter().zip([0, 2, 1]) { + for ((_, expr), expected) in project.exprs.iter().zip([0, 2, 1]) { assert!( matches!(arena.expression(*expr), ScalarExpression::ColumnRef { position, .. } if *position == expected) ); diff --git a/src/optimizer/rule/normalization/mod.rs b/src/optimizer/rule/normalization/mod.rs index d253dc1a..bbb2ac84 100644 --- a/src/optimizer/rule/normalization/mod.rs +++ b/src/optimizer/rule/normalization/mod.rs @@ -59,7 +59,7 @@ mod top_k; pub(crate) use compilation_in_advance::evaluator_bind_current; pub(crate) use elimination::{ apply_annotated_post_rules, apply_scan_order_hint, EliminateIndexFilter, EliminateProject, - OrderHintKind, ScanOrderHint, + MapProjectColumns, OrderHintKind, ScanOrderHint, }; pub(crate) use parameterized_index::{ParameterizeInnerJoin, ParameterizeMarkApply}; pub(crate) use simplification::constant_calculation_current; @@ -70,6 +70,7 @@ pub enum NormalizationRuleImpl { SkipResidualFilterColumns, // Combine operators EliminateProject, + MapProjectColumns, CollapseGroupByAgg, CombineFilter, // PushDown limit @@ -191,7 +192,9 @@ impl NormalizationRuleImpl { | NormalizationRuleImpl::SkipResidualFilterColumns => NormalizationRuleRootTag::Any, NormalizationRuleImpl::EliminateProject => NormalizationRuleRootTag::Any, NormalizationRuleImpl::CollapseGroupByAgg => NormalizationRuleRootTag::Aggregate, - NormalizationRuleImpl::GuardMarkApply => NormalizationRuleRootTag::Project, + NormalizationRuleImpl::GuardMarkApply | NormalizationRuleImpl::MapProjectColumns => { + NormalizationRuleRootTag::Project + } NormalizationRuleImpl::CombineFilter => NormalizationRuleRootTag::Filter, NormalizationRuleImpl::LimitProjectTranspose | NormalizationRuleImpl::PushLimitThroughJoin @@ -233,6 +236,9 @@ impl NormalizationRule for NormalizationRuleImpl { } NormalizationRuleImpl::GuardMarkApply => GuardMarkApply.apply(plan, arena, loader), NormalizationRuleImpl::EliminateProject => EliminateProject.apply(plan, arena, loader), + NormalizationRuleImpl::MapProjectColumns => { + MapProjectColumns.apply(plan, arena, loader) + } NormalizationRuleImpl::CollapseGroupByAgg => { CollapseGroupByAgg.apply(plan, arena, loader) } diff --git a/src/optimizer/rule/normalization/parameterized_index.rs b/src/optimizer/rule/normalization/parameterized_index.rs index 937356ec..58c7927c 100644 --- a/src/optimizer/rule/normalization/parameterized_index.rs +++ b/src/optimizer/rule/normalization/parameterized_index.rs @@ -173,7 +173,7 @@ fn parameterize_right_subtree( ) -> Result, DatabaseError> { match (&mut plan.operator, plan.childrens.as_mut()) { (Operator::Project(project), Childrens::Only(child)) => { - let Some(mut expr) = project.exprs.iter().copied().find(|expr| { + let Some(mut expr) = project.exprs.iter().map(|(_, expr)| *expr).find(|expr| { let column = expr.output_column_ref(arena); arena.same_column(column, *right_column) }) else { @@ -561,13 +561,10 @@ impl NormalizationRule for ParameterizeInnerJoin { i - old_left_len }; arena.alloc_expression(ScalarExpression::column_expr(column, position)) - }) - .collect(); + }); + let project = ProjectOperator::new(exprs); std::mem::swap(left, right); - let mut project = LogicalPlan::new( - Operator::Project(ProjectOperator::new(exprs)), - Childrens::None, - ); + let mut project = LogicalPlan::new(Operator::Project(project), Childrens::None); project.physical_option = Some(PhysicalOption::new( PlanImpl::Project, SortOption::Follow, @@ -1340,7 +1337,7 @@ mod inner_join_tests { project_op .exprs .iter() - .map(|expr| position(*expr, &arena)) + .map(|(_, expr)| position(*expr, &arena)) .collect::>(), vec![1, 2, 3, 0] ); diff --git a/src/optimizer/rule/normalization/simplification.rs b/src/optimizer/rule/normalization/simplification.rs index 2c793b95..c5c3de4f 100644 --- a/src/optimizer/rule/normalization/simplification.rs +++ b/src/optimizer/rule/normalization/simplification.rs @@ -171,13 +171,13 @@ mod test { if let Operator::Project(project_op) = best_plan.clone().operator { let constant_expr = ScalarExpression::Constant(DataValue::Int32(3)); if let ScalarExpression::Binary { right_expr, .. } = - arena.expression(project_op.exprs[0]) + arena.expression(project_op.exprs[0].1) { assert_eq!(arena.expression(*right_expr), &constant_expr); } else { unreachable!(); } - assert_eq!(arena.expression(project_op.exprs[1]), &constant_expr); + assert_eq!(arena.expression(project_op.exprs[1].1), &constant_expr); } else { unreachable!(); } @@ -224,11 +224,11 @@ mod test { if let Operator::Project(project_op) = best_plan.operator { assert_eq!( - arena.expression(project_op.exprs[0]), + arena.expression(project_op.exprs[0].1), &ScalarExpression::Constant(DataValue::Int32(1)) ); assert_eq!( - arena.expression(project_op.exprs[1]), + arena.expression(project_op.exprs[1].1), &ScalarExpression::Constant(DataValue::Int32(3)) ); } else { diff --git a/src/planner/mod.rs b/src/planner/mod.rs index 50e73dd2..24fc5ae2 100644 --- a/src/planner/mod.rs +++ b/src/planner/mod.rs @@ -365,7 +365,7 @@ impl LogicalPlan { Operator::Project(op) => op .exprs .iter() - .map(|expr| expr.output_column_ref(arena)) + .map(|(_, expr)| expr.output_column_ref(arena)) .collect(), Operator::TableScan(op) => op.columns().collect(), Operator::FunctionScan(op) => { diff --git a/src/planner/operator/mod.rs b/src/planner/operator/mod.rs index 28753450..34e1c919 100644 --- a/src/planner/operator/mod.rs +++ b/src/planner/operator/mod.rs @@ -835,7 +835,7 @@ impl Operator { op: &'operator ProjectOperator, _children: &mut ChildrenVisit<'operator>, ) -> Result<(), DatabaseError> { - for expr in &op.exprs { + for (_, expr) in &op.exprs { ExprVisitor::visit(self, *expr, self.arena)?; } Ok(()) diff --git a/src/planner/operator/project.rs b/src/planner/operator/project.rs index 00140a7a..ab9b9697 100644 --- a/src/planner/operator/project.rs +++ b/src/planner/operator/project.rs @@ -18,12 +18,14 @@ use kite_sql_serde_macros::ReferenceSerialization; #[derive(Debug, PartialEq, Eq, Clone, Hash, ReferenceSerialization)] pub struct ProjectOperator { - pub exprs: Vec, + pub exprs: Vec<(Option, ExprRef)>, } impl ProjectOperator { - pub fn new(exprs: Vec) -> Self { - Self { exprs } + pub fn new(exprs: impl IntoIterator) -> Self { + Self { + exprs: exprs.into_iter().map(|expr| (None, expr)).collect(), + } } } @@ -34,7 +36,7 @@ impl Explain for ProjectOperator { f: &mut std::fmt::Formatter<'_>, ) -> std::fmt::Result { f.write_str("Projection [")?; - fmt_explain_list(&self.exprs, ", ", arena, f)?; + fmt_explain_list(self.exprs.iter().map(|(_, expr)| expr), ", ", arena, f)?; f.write_str("]") } } diff --git a/src/planner/operator/visitor.rs b/src/planner/operator/visitor.rs index 190bdcb9..c772a66d 100644 --- a/src/planner/operator/visitor.rs +++ b/src/planner/operator/visitor.rs @@ -434,7 +434,7 @@ impl<'a, V: ExprVisitor, A: MetaArena + ?Sized> OperatorVisitor<'a> op: &'a ProjectOperator, _children: &mut ChildrenVisit<'a>, ) -> Result<(), DatabaseError> { - for expr in &op.exprs { + for (_, expr) in &op.exprs { ExprVisitor::visit(self.visitor, *expr, self.arena)?; } Ok(()) diff --git a/src/planner/operator/visitor_mut.rs b/src/planner/operator/visitor_mut.rs index 3b5a99d5..f7c17f9b 100644 --- a/src/planner/operator/visitor_mut.rs +++ b/src/planner/operator/visitor_mut.rs @@ -451,7 +451,7 @@ impl<'a, V: ExprVisitorMut, A: MetaArena + ?Sized> OperatorVisitorMut<'a> op: &'a mut ProjectOperator, _children: &mut ChildrenVisitMut<'a>, ) -> Result<(), DatabaseError> { - for expr in &mut op.exprs { + for (_, expr) in &mut op.exprs { ExprVisitorMut::visit(self.visitor, expr, self.arena)?; } Ok(()) diff --git a/src/storage/mod.rs b/src/storage/mod.rs index e920c3e4..419fb0ab 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1984,7 +1984,8 @@ impl ScanColumn { pub(crate) fn is_read(&self) -> bool { !matches!( self.deserializer, - TupleValueSerializableImpl::SkipFixed(_) | TupleValueSerializableImpl::SkipVariable + TupleValueSerializableImpl::SkipFixed { null: false, .. } + | TupleValueSerializableImpl::SkipVariable { null: false } ) } } @@ -2035,12 +2036,38 @@ impl TupleDeserializers { if forced || keep(position, scan_column.column) { read_len = index + 1; } else { - scan_column.deserializer = table_column.datatype().skip_serializable(); + scan_column.deserializer = table_column.datatype().skip_serializable(false); } } self.columns.truncate(read_len); } + pub(crate) fn null_unread( + &mut self, + arena: &A, + mut keep: impl FnMut(ColumnRef) -> bool, + ) -> bool { + let with_pk = self.pk_ty.is_some(); + let mut changed = false; + for scan_column in self.columns.iter_mut().filter(|column| column.is_read()) { + let table_column = arena.column(scan_column.column); + if scan_column.pinned + || (with_pk && table_column.desc().primary().is_some()) + || matches!( + scan_column.deserializer, + TupleValueSerializableImpl::SkipFixed { .. } + | TupleValueSerializableImpl::SkipVariable { .. } + ) + || keep(scan_column.column) + { + continue; + } + scan_column.deserializer = table_column.datatype().skip_serializable(true); + changed = true; + } + changed + } + pub(crate) fn unpin(&mut self) { for column in &mut self.columns { column.pinned = false; diff --git a/src/types/serialize.rs b/src/types/serialize.rs index be8bb3b3..fba66576 100644 --- a/src/types/serialize.rs +++ b/src/types/serialize.rs @@ -22,7 +22,9 @@ use ordered_float::OrderedFloat; #[cfg(feature = "decimal")] use rust_decimal::Decimal; use std::fmt::Debug; -use std::io::{Cursor, Read, Seek, SeekFrom, Write}; +#[cfg(feature = "decimal")] +use std::io::Read; +use std::io::{Cursor, Seek, SeekFrom, Write}; use std::str; macro_rules! impl_tuple_value_serializable { @@ -103,8 +105,13 @@ pub enum TupleValueSerializableImpl { zone: bool, }, Decimal, - SkipFixed(usize), - SkipVariable, + SkipFixed { + len: usize, + null: bool, + }, + SkipVariable { + null: bool, + }, Varbinary { len: Option, }, @@ -149,8 +156,14 @@ impl TupleValueSerializable for TupleValueSerializableImpl { } .to_raw(value, writer), TupleValueSerializableImpl::Decimal => DecimalSerializable.to_raw(value, writer), - TupleValueSerializableImpl::SkipFixed(len) => SkipFixed(*len).to_raw(value, writer), - TupleValueSerializableImpl::SkipVariable => SkipVariable.to_raw(value, writer), + TupleValueSerializableImpl::SkipFixed { len, null } => SkipFixed { + len: *len, + null: *null, + } + .to_raw(value, writer), + TupleValueSerializableImpl::SkipVariable { null } => { + SkipVariable { null: *null }.to_raw(value, writer) + } } } @@ -227,11 +240,13 @@ impl TupleValueSerializable for TupleValueSerializableImpl { TupleValueSerializableImpl::Decimal => { DecimalSerializable.filling_value(reader, values, pool, scope) } - TupleValueSerializableImpl::SkipFixed(len) => { - SkipFixed(*len).filling_value(reader, values, pool, scope) + TupleValueSerializableImpl::SkipFixed { len, null } => SkipFixed { + len: *len, + null: *null, } - TupleValueSerializableImpl::SkipVariable => { - SkipVariable.filling_value(reader, values, pool, scope) + .filling_value(reader, values, pool, scope), + TupleValueSerializableImpl::SkipVariable { null } => { + SkipVariable { null: *null }.filling_value(reader, values, pool, scope) } } } @@ -397,9 +412,14 @@ struct TimeStampSerializable { struct DecimalSerializable; #[derive(Debug)] -struct SkipFixed(usize); +struct SkipFixed { + len: usize, + null: bool, +} #[derive(Debug)] -struct SkipVariable; +struct SkipVariable { + null: bool, +} fn write_spaces(writer: &mut W, mut len: usize) -> Result<(), DatabaseError> { const SPACES: [u8; 64] = [b' '; 64]; @@ -418,10 +438,16 @@ macro_rules! define_le_io { writer.write_all(&value.to_le_bytes()) } - fn $read_fn(reader: &mut R) -> std::io::Result<$ty> { - let mut bytes = [0u8; std::mem::size_of::<$ty>()]; - reader.read_exact(&mut bytes)?; - Ok(<$ty>::from_le_bytes(bytes)) + fn $read_fn(reader: &mut Cursor<&[u8]>) -> std::io::Result<$ty> { + // Straight from the slice: `Read::read_exact` costs a call per value. + let start = reader.position() as usize; + let bytes = reader + .get_ref() + .get(start..) + .and_then(|rest| rest.first_chunk::<{ std::mem::size_of::<$ty>() }>()) + .ok_or(std::io::ErrorKind::UnexpectedEof)?; + reader.set_position((start + bytes.len()) as u64); + Ok(<$ty>::from_le_bytes(*bytes)) } )* }; @@ -735,11 +761,14 @@ impl TupleValueSerializable for SkipFixed { unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, - _: &mut Vec, + values: &mut Vec, _: &mut BytesPool, _: Option<&MappedScope>, ) -> Result<(), DatabaseError> { - reader.seek(SeekFrom::Current(self.0 as i64))?; + reader.seek(SeekFrom::Current(self.len as i64))?; + if self.null { + values.push(DataValue::Null); + } Ok(()) } } @@ -752,21 +781,25 @@ impl TupleValueSerializable for SkipVariable { unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, - _: &mut Vec, + values: &mut Vec, _: &mut BytesPool, _: Option<&MappedScope>, ) -> Result<(), DatabaseError> { let len = read_u32_le(reader)? as usize; reader.seek(SeekFrom::Current(len as i64))?; + if self.null { + values.push(DataValue::Null); + } Ok(()) } } impl LogicalType { - pub fn skip_serializable(&self) -> TupleValueSerializableImpl { - self.raw_len() - .map(TupleValueSerializableImpl::SkipFixed) - .unwrap_or(TupleValueSerializableImpl::SkipVariable) + pub fn skip_serializable(&self, null: bool) -> TupleValueSerializableImpl { + match self.raw_len() { + Some(len) => TupleValueSerializableImpl::SkipFixed { len, null }, + None => TupleValueSerializableImpl::SkipVariable { null }, + } } pub fn serializable(&self) -> TupleValueSerializableImpl { @@ -1000,7 +1033,10 @@ mod tests { }; assert_eq!(values, vec![value]); - let skip_fixed = TupleValueSerializableImpl::SkipFixed(2); + let skip_fixed = TupleValueSerializableImpl::SkipFixed { + len: 2, + null: false, + }; let mut values = vec![DataValue::Int32(1)]; let mut cursor = Cursor::new([1_u8, 2, 3].as_slice()); // SAFETY: without a scope every value is copied. @@ -1010,7 +1046,7 @@ mod tests { assert_eq!(values, vec![DataValue::Int32(1)]); assert_eq!(cursor.position(), 2); - let skip_variable = TupleValueSerializableImpl::SkipVariable; + let skip_variable = TupleValueSerializableImpl::SkipVariable { null: false }; let mut bytes = Vec::new(); write_u32_le(&mut bytes, 3)?; bytes.extend_from_slice(&[1, 2, 3, 4]); @@ -1038,16 +1074,22 @@ mod tests { TupleValueSerializableImpl::UInt64 ); assert_eq!( - LogicalType::Char(8, CharLengthUnits::Octets).skip_serializable(), - TupleValueSerializableImpl::SkipFixed(8) + LogicalType::Char(8, CharLengthUnits::Octets).skip_serializable(false), + TupleValueSerializableImpl::SkipFixed { + len: 8, + null: false + } ); assert_eq!( - LogicalType::Varchar(None, CharLengthUnits::Characters).skip_serializable(), - TupleValueSerializableImpl::SkipVariable + LogicalType::Varchar(None, CharLengthUnits::Characters).skip_serializable(true), + TupleValueSerializableImpl::SkipVariable { null: true } ); assert_eq!( - LogicalType::SqlNull.skip_serializable(), - TupleValueSerializableImpl::SkipFixed(0) + LogicalType::SqlNull.skip_serializable(false), + TupleValueSerializableImpl::SkipFixed { + len: 0, + null: false + } ); assert_panics(|| { let _ = LogicalType::SqlNull.serializable(); diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 0a0fc9e6..7ae11f86 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -782,8 +782,8 @@ impl Tuple { // pushing one here would shift every later projected column. if !matches!( deserializer.borrow(), - TupleValueSerializableImpl::SkipFixed(_) - | TupleValueSerializableImpl::SkipVariable + TupleValueSerializableImpl::SkipFixed { null: false, .. } + | TupleValueSerializableImpl::SkipVariable { null: false } ) { self.values.push(DataValue::Null); } @@ -1095,8 +1095,8 @@ mod tests { { let projection_serializers = vec![ columns[0].datatype().serializable(), - columns[1].datatype().skip_serializable(), - columns[2].datatype().skip_serializable(), + columns[1].datatype().skip_serializable(false), + columns[2].datatype().skip_serializable(false), columns[3].datatype().serializable(), ]; let mut tuple_2 = Tuple { From 5ccc2976cb9387ff8bc91f619d576a25aab4d99c Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 4 Nov 2026 20:02:01 +0800 Subject: [PATCH 07/14] docs(tpcc): refresh 720s TPC-C results on current code --- README.md | 6 +++--- tpcc/README.md | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b71f6c18..0515e18c 100755 --- a/README.md +++ b/README.md @@ -135,9 +135,9 @@ print(list(db.run("select * from demo"))) | Backend | TpmC | New-Order | Payment | Order-Status | Delivery | Stock-Level | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| KiteSQL LMDB | 209840 | 254 | 50 | 28 | 345 | 202 | -| KiteSQL RocksDB | 48049 | 354 | 85 | 146 | 14511 | 562 | -| SQLite (WAL, `synchronous=OFF`) | 192435 | 270 | 40 | 33 | 344 | 265 | +| KiteSQL LMDB | 214271 | 252 | 46 | 26 | 340 | 192 | +| KiteSQL RocksDB | 47415 | 360 | 79 | 144 | 15047 | 548 | +| SQLite (WAL, `synchronous=OFF`) | 185982 | 278 | 41 | 34 | 350 | 270 | 👉 [Details and how to reproduce](tpcc/README.md) diff --git a/tpcc/README.md b/tpcc/README.md index a19317e7..d6df38a7 100644 --- a/tpcc/README.md +++ b/tpcc/README.md @@ -72,11 +72,11 @@ Local 720-second comparison on the machine above: | Backend | TpmC | New-Order p90 (µs) | Payment p90 (µs) | Order-Status p90 (µs) | Delivery p90 (µs) | Stock-Level p90 (µs) | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| KiteSQL LMDB | 209840 | 254 | 50 | 28 | 345 | 202 | -| KiteSQL RocksDB | 48049 | 354 | 85 | 146 | 14511 | 562 | -| SQLite | 192435 | 270 | 40 | 33 | 344 | 265 | +| KiteSQL LMDB | 214271 | 252 | 46 | 26 | 340 | 192 | +| KiteSQL RocksDB | 47415 | 360 | 79 | 144 | 15047 | 548 | +| SQLite | 185982 | 278 | 41 | 34 | 350 | 270 | -- Run date: `2026-11-01`. Latency is measured in microseconds and includes commit. +- Run date: `2026-11-04`. Latency is measured in microseconds and includes commit. - Every row is pinned to `cpu8`, the highest-frequency P-core on this machine (5.4 GHz). - All rows use `--num-ware 1`, `--max-retry 5`, and TPCC's default 720-second measure time. - All rows run without `--sync`, so commits are not fsynced; SQLite uses WAL with `synchronous = OFF`. From 688f837a571455641ee79fb8f07a69d3cfd5d18d Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 4 Nov 2026 21:02:49 +0800 Subject: [PATCH 08/14] perf: trust stored strings as UTF-8 and run read-only TPC-C transactions read-only Strings decoded from stored rows are no longer validated as UTF-8 outside debug builds: every stored string was encoded from a `str`. Order-Status and Stock-Level now run in read-only transactions, as TPC-C allows; SQLite starts them with a deferred BEGIN. --- src/storage/table_codec.rs | 1 + src/types/serialize.rs | 30 +++++++++++++++-------------- src/types/tuple.rs | 1 + tpcc/src/backend/dual.rs | 6 +++--- tpcc/src/backend/kitesql_lmdb.rs | 11 +++++++---- tpcc/src/backend/kitesql_rocksdb.rs | 11 +++++++---- tpcc/src/backend/mod.rs | 2 +- tpcc/src/backend/sqlite.rs | 12 ++++++++---- tpcc/src/main.rs | 3 ++- 9 files changed, 46 insertions(+), 31 deletions(-) diff --git a/src/storage/table_codec.rs b/src/storage/table_codec.rs index b448a03a..c3a6eb1d 100644 --- a/src/storage/table_codec.rs +++ b/src/storage/table_codec.rs @@ -834,6 +834,7 @@ impl TableCodec { /// /// # Safety /// + /// `bytes` must be a row this crate encoded, since strings are not checked again as UTF-8. /// With a `scope`, `bytes` must be a read of the transaction holding it. #[inline] pub(crate) unsafe fn decode_tuple_into( diff --git a/src/types/serialize.rs b/src/types/serialize.rs index fba66576..030860c7 100644 --- a/src/types/serialize.rs +++ b/src/types/serialize.rs @@ -64,7 +64,8 @@ pub(crate) trait TupleValueSerializable: Debug { /// /// # Safety /// - /// With a `scope`, those bytes must be a read of the transaction holding it. + /// Those bytes must be a value this crate encoded, since strings are not checked again as + /// UTF-8. With a `scope`, they must be a read of the transaction holding it. unsafe fn filling_value( &self, reader: &mut Cursor<&[u8]>, @@ -282,17 +283,18 @@ impl BytesPool { /// # Safety /// - /// With a `scope`, `bytes` must be a read of the transaction holding it. - pub(crate) unsafe fn string( - &mut self, - bytes: &[u8], - scope: Option<&MappedScope>, - ) -> Result { - Ok(match scope { + /// `bytes` must be UTF-8, as every string this crate encodes is, and is not checked again + /// outside debug builds. With a `scope`, `bytes` must be a read of the transaction holding it. + pub(crate) unsafe fn string(&mut self, bytes: &[u8], scope: Option<&MappedScope>) -> Utf8Str { + debug_assert!(str::from_utf8(bytes).is_ok(), "stored string is not UTF-8"); + // SAFETY: the caller guarantees `bytes` is UTF-8. + let value = unsafe { str::from_utf8_unchecked(bytes) }; + match scope { // SAFETY: the caller guarantees `bytes` is a read of `scope`'s transaction. - Some(scope) => unsafe { Utf8Str::mapped(str::from_utf8(bytes)?, scope) }, - None => String::from_utf8(self.bytes(bytes))?.into(), - }) + Some(scope) => unsafe { Utf8Str::mapped(value, scope) }, + // SAFETY: the buffer holds exactly `bytes`, which are UTF-8. + None => unsafe { String::from_utf8_unchecked(self.bytes(bytes)) }.into(), + } } /// # Safety @@ -588,7 +590,7 @@ impl TupleValueSerializable for CharSerializable { values.push(DataValue::Utf8 { // SAFETY: the caller's guarantee covers `bytes`. - value: unsafe { pool.string(&bytes[..trimmed_len], scope)? }, + value: unsafe { pool.string(&bytes[..trimmed_len], scope) }, ty: Utf8Type::Fixed(self.len), unit: self.unit, }); @@ -623,7 +625,7 @@ impl TupleValueSerializable for VarcharSerializable { let len = read_u32_le(reader)? as usize; values.push(DataValue::Utf8 { // SAFETY: the caller's guarantee covers the bytes under `reader`. - value: unsafe { pool.string(read_slice(reader, len)?, scope)? }, + value: unsafe { pool.string(read_slice(reader, len)?, scope) }, ty: Utf8Type::Variable(self.len), unit: self.unit, }); @@ -872,7 +874,7 @@ mod tests { fn bytes_pool_reuses_recycled_buffers() -> Result<(), DatabaseError> { let mut pool = BytesPool::default(); // SAFETY: without a scope the string is copied. - let value = unsafe { pool.string(b"abc", None)? }; + let value = unsafe { pool.string(b"abc", None) }; let buffer = value.as_ptr(); let large = vec![0; BytesPool::MAX_BUFFER_LEN + 1]; let mut values = vec![ diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 7ae11f86..1a05c9bf 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -752,6 +752,7 @@ impl Tuple { /// /// # Safety /// + /// `bytes` must be a row this crate encoded, since strings are not checked again as UTF-8. /// With a `scope`, `bytes` must be a read of the transaction holding it. #[inline] pub(crate) unsafe fn deserialize_from_into( diff --git a/tpcc/src/backend/dual.rs b/tpcc/src/backend/dual.rs index 30b25e54..df89d1a5 100644 --- a/tpcc/src/backend/dual.rs +++ b/tpcc/src/backend/dual.rs @@ -93,10 +93,10 @@ impl BackendControl for DualBackend { Err(TpccError::InvalidBackend) } - fn new_transaction(&self) -> Result, TpccError> { + fn new_transaction(&self, read_only: bool) -> Result, TpccError> { Ok(DualTransaction { - kitesql: self.kitesql.new_transaction()?, - sqlite: self.sqlite.new_transaction()?, + kitesql: self.kitesql.new_transaction(read_only)?, + sqlite: self.sqlite.new_transaction(read_only)?, }) } diff --git a/tpcc/src/backend/kitesql_lmdb.rs b/tpcc/src/backend/kitesql_lmdb.rs index f10d1e87..b2b11cf4 100644 --- a/tpcc/src/backend/kitesql_lmdb.rs +++ b/tpcc/src/backend/kitesql_lmdb.rs @@ -51,9 +51,12 @@ impl KiteSqlLmdbBackend { Ok(groups) } - fn start_transaction(&self) -> Result, TpccError> { + fn start_transaction( + &self, + read_only: bool, + ) -> Result, TpccError> { Ok(KiteSqlLmdbTransactionWrapper { - inner: self.database.new_transaction(false)?, + inner: self.database.new_transaction(read_only)?, }) } } @@ -81,8 +84,8 @@ impl BackendControl for KiteSqlLmdbBackend { where Self: 'a; - fn new_transaction(&self) -> Result, TpccError> { - self.start_transaction() + fn new_transaction(&self, read_only: bool) -> Result, TpccError> { + self.start_transaction(read_only) } fn worker(&self) -> Result, TpccError> { diff --git a/tpcc/src/backend/kitesql_rocksdb.rs b/tpcc/src/backend/kitesql_rocksdb.rs index 8025a834..6dda6fc8 100644 --- a/tpcc/src/backend/kitesql_rocksdb.rs +++ b/tpcc/src/backend/kitesql_rocksdb.rs @@ -72,9 +72,12 @@ impl KiteSqlRocksBackend { Ok(groups) } - fn start_transaction(&self) -> Result, TpccError> { + fn start_transaction( + &self, + read_only: bool, + ) -> Result, TpccError> { Ok(KiteSqlRocksTransaction { - inner: self.database.new_transaction(false)?, + inner: self.database.new_transaction(read_only)?, }) } } @@ -102,8 +105,8 @@ impl BackendControl for KiteSqlRocksBackend { where Self: 'a; - fn new_transaction(&self) -> Result, TpccError> { - self.start_transaction() + fn new_transaction(&self, read_only: bool) -> Result, TpccError> { + self.start_transaction(read_only) } fn worker(&self) -> Result, TpccError> { diff --git a/tpcc/src/backend/mod.rs b/tpcc/src/backend/mod.rs index e1ffdb48..58f4efc3 100644 --- a/tpcc/src/backend/mod.rs +++ b/tpcc/src/backend/mod.rs @@ -46,7 +46,7 @@ pub trait BackendControl: SimpleExecutor { where Self: 'a; - fn new_transaction(&self) -> Result, TpccError>; + fn new_transaction(&self, read_only: bool) -> Result, TpccError>; fn worker(&self) -> Result, TpccError>; diff --git a/tpcc/src/backend/sqlite.rs b/tpcc/src/backend/sqlite.rs index 576ab3e3..e11b1cf8 100644 --- a/tpcc/src/backend/sqlite.rs +++ b/tpcc/src/backend/sqlite.rs @@ -66,8 +66,12 @@ impl SqliteBackend { Ok(groups) } - fn start_transaction(&self) -> Result, TpccError> { - self.connection.execute("BEGIN IMMEDIATE")?; + fn start_transaction(&self, read_only: bool) -> Result, TpccError> { + self.connection.execute(if read_only { + "BEGIN" + } else { + "BEGIN IMMEDIATE" + })?; Ok(SqliteTransaction { connection: &self.connection, finished: false, @@ -98,8 +102,8 @@ impl BackendControl for SqliteBackend { where Self: 'a; - fn new_transaction(&self) -> Result, TpccError> { - self.start_transaction() + fn new_transaction(&self, read_only: bool) -> Result, TpccError> { + self.start_transaction(read_only) } fn worker(&self) -> Result, TpccError> { diff --git a/tpcc/src/main.rs b/tpcc/src/main.rs index f56d0359..588751ea 100644 --- a/tpcc/src/main.rs +++ b/tpcc/src/main.rs @@ -62,6 +62,7 @@ pub(crate) const RT_LIMITS: [Duration; 5] = [ Duration::from_secs(8), Duration::from_secs(2), ]; +const TX_READ_ONLY: [bool; 5] = [false, false, true, false, true]; const TX_NAMES: [&str; 5] = [ "New-Order", "Payment", @@ -266,7 +267,7 @@ fn measure( let mut last_error = None; for _attempt in 0..=args.max_retry { let transaction_start = Instant::now(); - let mut tx = backend.new_transaction()?; + let mut tx = backend.new_transaction(TX_READ_ONLY[i])?; if let Err(err) = do_tpcc_transaction::(i, rng, &mut tx, args.num_ware, &tpcc_args, statement) From 539b868876f5f1d7426e7b909176f63c04260a01 Mon Sep 17 00:00:00 2001 From: kould Date: Wed, 4 Nov 2026 23:37:55 +0800 Subject: [PATCH 09/14] perf(execution): keep mapped strings mapped in materializing operators Operators that keep values while they run cloned them, and cloning a mapped string copies it out of the read transaction. They now copy with `DataValue::clone_in_txn`, which leaves a mapped value pointing into its read: grouping, distinct, sort, window and join keys, joined rows of hash and nested-loop joins, MIN/MAX and DISTINCT aggregate state, IN-subquery keys, index probes and set-operation rows. Each drops what it kept when `reset` runs before the next execution. `Clone` still copies, so values that leave the engine stay owned. --- src/execution/dql/aggregate/avg.rs | 3 +- src/execution/dql/aggregate/count.rs | 3 +- src/execution/dql/aggregate/group_concat.rs | 3 +- src/execution/dql/aggregate/hash_agg.rs | 9 +- src/execution/dql/aggregate/min_max.rs | 3 +- src/execution/dql/aggregate/stream_agg.rs | 9 +- src/execution/dql/aggregate/sum.rs | 3 +- src/execution/dql/aggregate/total.rs | 3 +- src/execution/dql/join/hash/mod.rs | 14 ++- src/execution/dql/join/hash_join.rs | 10 +- src/execution/dql/join/index_probe_join.rs | 7 +- src/execution/dql/join/nested_loop_join.rs | 8 +- src/execution/dql/mark_apply.rs | 10 +- src/execution/dql/set_membership.rs | 4 +- src/execution/dql/sort.rs | 3 +- src/execution/dql/window.rs | 8 +- src/execution/dql/window/function.rs | 3 +- src/storage/lmdb.rs | 101 ++++++++++++++++++++ src/types/mapped.rs | 10 +- src/types/value.rs | 33 +++++++ 20 files changed, 204 insertions(+), 43 deletions(-) diff --git a/src/execution/dql/aggregate/avg.rs b/src/execution/dql/aggregate/avg.rs index e3b10a06..8308b773 100644 --- a/src/execution/dql/aggregate/avg.rs +++ b/src/execution/dql/aggregate/avg.rs @@ -45,7 +45,8 @@ impl Accumulator for AvgAccumulator { fn update_value(&mut self, value: &DataValue) -> Result<(), DatabaseError> { if !value.is_null() { if let Some(values) = &mut self.distinct_values { - if !values.insert(value.clone()) { + // SAFETY: an executor resets its accumulators before it runs again. + if !values.insert(unsafe { value.clone_in_txn() }) { return Ok(()); } } diff --git a/src/execution/dql/aggregate/count.rs b/src/execution/dql/aggregate/count.rs index 622c2505..97d794d9 100644 --- a/src/execution/dql/aggregate/count.rs +++ b/src/execution/dql/aggregate/count.rs @@ -72,7 +72,8 @@ impl DistinctCountAccumulator { impl Accumulator for DistinctCountAccumulator { fn update_value(&mut self, value: &DataValue) -> Result<(), DatabaseError> { if !value.is_null() && !self.distinct_values.contains(value) { - self.distinct_values.insert(value.clone()); + // SAFETY: an executor resets its accumulators before it runs again. + self.distinct_values.insert(unsafe { value.clone_in_txn() }); self.result = DataValue::Int32(self.distinct_values.len() as i32); } diff --git a/src/execution/dql/aggregate/group_concat.rs b/src/execution/dql/aggregate/group_concat.rs index ff77457c..eb2fa522 100644 --- a/src/execution/dql/aggregate/group_concat.rs +++ b/src/execution/dql/aggregate/group_concat.rs @@ -41,7 +41,8 @@ impl Accumulator for GroupConcatAccumulator { return Ok(()); } if let Some(values) = &mut self.distinct_values { - if !values.insert(value.clone()) { + // SAFETY: an executor resets its accumulators before it runs again. + if !values.insert(unsafe { value.clone_in_txn() }) { return Ok(()); } } diff --git a/src/execution/dql/aggregate/hash_agg.rs b/src/execution/dql/aggregate/hash_agg.rs index 4c94a252..9d0e4700 100644 --- a/src/execution/dql/aggregate/hash_agg.rs +++ b/src/execution/dql/aggregate/hash_agg.rs @@ -102,12 +102,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for HashAggExecutor<'a> // Evaluate the key in place, and drop it again if its group exists. let start = self.keys.len(); for expr in self.groupby_exprs { - self.keys.push( - plan_arena - .expression(*expr) - .eval(plan_arena, Some(tuple))? - .into_owned(), - ); + let key = plan_arena.expression(*expr).eval(plan_arena, Some(tuple))?; + // SAFETY: the keys live until `reset` clears them, before the next execution. + self.keys.push(unsafe { DataValue::own_in_txn(key) }); } let keys = &self.keys; diff --git a/src/execution/dql/aggregate/min_max.rs b/src/execution/dql/aggregate/min_max.rs index 48b8bf1d..c1aacf8c 100644 --- a/src/execution/dql/aggregate/min_max.rs +++ b/src/execution/dql/aggregate/min_max.rs @@ -55,7 +55,8 @@ impl Accumulator for MinMaxAccumulator { } else { true } - .then(|| self.result = value.clone()); + // SAFETY: an executor resets its accumulators before it runs again. + .then(|| self.result = unsafe { value.clone_in_txn() }); } Ok(()) diff --git a/src/execution/dql/aggregate/stream_agg.rs b/src/execution/dql/aggregate/stream_agg.rs index f63d3138..0f36eee9 100644 --- a/src/execution/dql/aggregate/stream_agg.rs +++ b/src/execution/dql/aggregate/stream_agg.rs @@ -113,12 +113,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for StreamAggExecutor<' let tuple = arena.result_tuple(); for expr in self.groupby_exprs { - self.group_keys.push( - plan_arena - .expression(*expr) - .eval(plan_arena, Some(tuple))? - .into_owned(), - ); + let key = plan_arena.expression(*expr).eval(plan_arena, Some(tuple))?; + // SAFETY: the keys live until `reset` clears them, before the next execution. + self.group_keys.push(unsafe { DataValue::own_in_txn(key) }); } if !self.in_group { diff --git a/src/execution/dql/aggregate/sum.rs b/src/execution/dql/aggregate/sum.rs index 77adcfb0..6c3aeefe 100644 --- a/src/execution/dql/aggregate/sum.rs +++ b/src/execution/dql/aggregate/sum.rs @@ -125,7 +125,8 @@ impl DistinctSumAccumulator { impl Accumulator for DistinctSumAccumulator { fn update_value(&mut self, value: &DataValue) -> Result<(), DatabaseError> { if !self.distinct_values.contains(value) { - self.distinct_values.insert(value.clone()); + // SAFETY: an executor resets its accumulators before it runs again. + self.distinct_values.insert(unsafe { value.clone_in_txn() }); self.inner.update_value(value)?; } diff --git a/src/execution/dql/aggregate/total.rs b/src/execution/dql/aggregate/total.rs index 0467a04c..5abcd488 100644 --- a/src/execution/dql/aggregate/total.rs +++ b/src/execution/dql/aggregate/total.rs @@ -41,7 +41,8 @@ impl Accumulator for TotalAccumulator { return Ok(()); } if let Some(values) = &mut self.distinct_values { - if !values.insert(value.clone()) { + // SAFETY: an executor resets its accumulators before it runs again. + if !values.insert(unsafe { value.clone_in_txn() }) { return Ok(()); } } diff --git a/src/execution/dql/join/hash/mod.rs b/src/execution/dql/join/hash/mod.rs index 20efafae..fc26eac1 100644 --- a/src/execution/dql/join/hash/mod.rs +++ b/src/execution/dql/join/hash/mod.rs @@ -135,20 +135,24 @@ pub(crate) fn filter( } } +/// Joins `build` with `probe`. The joined row passes through this execution like the rows it +/// joins, so their mapped strings stay mapped in it. pub(crate) fn join_rows( build: &TupleImpl, probe: &mut TupleImpl, consume_probe: bool, ) -> TupleImpl { + // SAFETY: the joined row is read in this execution, as the rows it copies from are. + let copy = |value: &DataValue| unsafe { value.clone_in_txn() }; if consume_probe { - let pk = build.pk().cloned().or_else(|| probe.take_pk()); + let pk = build.pk().map(copy).or_else(|| probe.take_pk()); let mut row = std::mem::take(probe); - row.prepend(build.iter().cloned()); + row.prepend(build.iter().map(copy)); row.set_pk(pk); return row; } let mut values = Vec::with_capacity(build.len() + probe.len()); - values.extend(build.iter().cloned()); - values.extend(probe.iter().cloned()); - Tuple::new(build.pk().or(probe.pk()).cloned(), values).into() + values.extend(build.iter().map(copy)); + values.extend(probe.iter().map(copy)); + Tuple::new(build.pk().or(probe.pk()).map(copy), values).into() } diff --git a/src/execution/dql/join/hash_join.rs b/src/execution/dql/join/hash_join.rs index 85b40ec4..4dc3c28b 100644 --- a/src/execution/dql/join/hash_join.rs +++ b/src/execution/dql/join/hash_join.rs @@ -139,12 +139,10 @@ impl<'a> HashJoin<'a> { plan_arena: &(dyn MetaArena + '_), ) -> Result<(), DatabaseError> { for expr in on_keys { - buf.push( - plan_arena - .expression(*expr) - .eval(plan_arena, Some(tuple))? - .into_owned(), - ); + let key = plan_arena.expression(*expr).eval(plan_arena, Some(tuple))?; + // SAFETY: the build table and probe keys are dropped when `reset` returns to + // `HashJoinState::Build`, before the next execution. + buf.push(unsafe { DataValue::own_in_txn(key) }); } Ok(()) } diff --git a/src/execution/dql/join/index_probe_join.rs b/src/execution/dql/join/index_probe_join.rs index 316e93b4..7a07cd75 100644 --- a/src/execution/dql/join/index_probe_join.rs +++ b/src/execution/dql/join/index_probe_join.rs @@ -21,6 +21,7 @@ use crate::planner::operator::index_probe_join::IndexProbeJoinOperator; use crate::planner::{ExecMetaArena, LogicalPlan, MetaArena}; use crate::storage::StorageTransaction; use crate::types::index::RuntimeIndexProbe; +use crate::types::value::DataValue; pub struct IndexProbeJoin<'a> { op: &'a IndexProbeJoinOperator, @@ -64,8 +65,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexProbeJoin<'a> let mut left = arena.take_tuple(); let key = plan_arena .expression(self.op.probe) - .eval(plan_arena, Some(&left))? - .into_owned(); + .eval(plan_arena, Some(&left))?; + // SAFETY: the probe is popped by the right input during this execution, and the + // probe stack is cleared before the next one. + let key = unsafe { DataValue::own_in_txn(key) }; if key.has_null() { arena.release(left); continue; diff --git a/src/execution/dql/join/nested_loop_join.rs b/src/execution/dql/join/nested_loop_join.rs index c097d432..8b6c32c8 100644 --- a/src/execution/dql/join/nested_loop_join.rs +++ b/src/execution/dql/join/nested_loop_join.rs @@ -329,13 +329,15 @@ impl NestedLoopJoin<'_> { /// `right`: current inner tuple (logical left for RightOuter). /// `ty`: the type of join fn emit_tuple(left: &TupleImpl, right: &mut TupleImpl, ty: JoinType) { - let values = left.iter().cloned(); + // SAFETY: the joined row is read in this execution, as the rows it copies from are. + let copy = |value: &DataValue| unsafe { value.clone_in_txn() }; + let values = left.iter().map(copy); let pk = if matches!(ty, JoinType::RightOuter) { right.extend(values); - right.take_pk().or_else(|| left.pk().cloned()) + right.take_pk().or_else(|| left.pk().map(copy)) } else { right.prepend(values); - left.pk().cloned().or_else(|| right.take_pk()) + left.pk().map(copy).or_else(|| right.take_pk()) }; right.set_pk(pk); } diff --git a/src/execution/dql/mark_apply.rs b/src/execution/dql/mark_apply.rs index dc48fecb..d9cb094e 100644 --- a/src/execution/dql/mark_apply.rs +++ b/src/execution/dql/mark_apply.rs @@ -196,7 +196,9 @@ impl<'a> MarkApply<'a> { plan_arena .expression(*probe) .eval(plan_arena, Some(left_tuple)) - .map(|value| value.into_owned()) + // SAFETY: the probe is popped by the right input during this execution, and + // the probe stack is cleared before the next one. + .map(|value| unsafe { DataValue::own_in_txn(value) }) }) .transpose() } @@ -281,8 +283,10 @@ impl<'a> MarkApply<'a> { let values = SplitTupleRef::new(left_tuple, arena.result_tuple()); let key = plan_arena .expression(right_expr) - .eval(plan_arena, Some(&values))? - .into_owned(); + .eval(plan_arena, Some(&values))?; + // SAFETY: the materialized keys are rebuilt when a new execution resets this node + // (see `reset`), so they are only read in the execution that read them. + let key = unsafe { DataValue::own_in_txn(key) }; is_empty = false; if key.is_null() { has_null = true; diff --git a/src/execution/dql/set_membership.rs b/src/execution/dql/set_membership.rs index b5577bc5..a6fd7cfb 100644 --- a/src/execution/dql/set_membership.rs +++ b/src/execution/dql/set_membership.rs @@ -91,7 +91,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SetMembership<'a> { continue; } let offset = self.right_rows.len(); - self.right_rows.extend(tuple.iter().cloned()); + // SAFETY: the rows live until `reset` clears them, before the next execution. + self.right_rows + .extend(tuple.iter().map(|value| unsafe { value.clone_in_txn() })); let (hasher, rows) = (&self.hasher, &self.right_rows); self.right_counts .insert_unique(hash, RightRow { offset, count: 1 }, |row| { diff --git a/src/execution/dql/sort.rs b/src/execution/dql/sort.rs index 80193172..25a2008e 100644 --- a/src/execution/dql/sort.rs +++ b/src/execution/dql/sort.rs @@ -95,7 +95,8 @@ pub(crate) fn sort_tuples( let value = plan_arena .expression(*expr) .eval(plan_arena, Some(&row.tuple))?; - eval_values.push(value.into_owned()); + // SAFETY: the keys are dropped when this sort returns. + eval_values.push(unsafe { DataValue::own_in_txn(value) }); } row.outer_values = plan_arena.take_outer_values(); } diff --git a/src/execution/dql/window.rs b/src/execution/dql/window.rs index 1061118a..bab68e6f 100644 --- a/src/execution/dql/window.rs +++ b/src/execution/dql/window.rs @@ -144,9 +144,13 @@ impl<'a> Window<'a> { } else if boundary.is_none() { boundary = Some(Boundary::Peer); } - self.state.sort_values[index] = value.into_owned(); + // SAFETY: the keys live until `reset` clears them, before the next execution. + self.state.sort_values[index] = unsafe { DataValue::own_in_txn(value) }; } else if !self.state.started { - self.state.sort_values.push(value.into_owned()); + // SAFETY: as above. + self.state + .sort_values + .push(unsafe { DataValue::own_in_txn(value) }); } } self.state.started = true; diff --git a/src/execution/dql/window/function.rs b/src/execution/dql/window/function.rs index 1558bf28..cb2c3c79 100644 --- a/src/execution/dql/window/function.rs +++ b/src/execution/dql/window/function.rs @@ -116,7 +116,8 @@ impl WindowFunction for Aggregate { accumulator.evaluate()?; let result = accumulator.result(); for (_, row) in &mut rows[peer] { - row[output_position] = result.clone(); + // SAFETY: the rows only pass through this execution, as the values they were read with. + row[output_position] = unsafe { result.clone_in_txn() }; } Ok(()) } diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs index 7e649278..a58beea1 100644 --- a/src/storage/lmdb.rs +++ b/src/storage/lmdb.rs @@ -608,6 +608,107 @@ mod tests { assert_eq!(value, "kite"); } + #[test] + fn materializing_operators_keep_strings_mapped_across_executions() { + let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + let mut db = DataBaseBuilder::path(temp_dir.path()).build_lmdb().unwrap(); + db.ddl("create table t (a int primary key, b varchar, c int)") + .unwrap(); + db.ddl("create table u (a int primary key, b varchar)") + .unwrap(); + db.run("insert into t values (1, 'kite', 1), (2, 'db', 1), (3, 'kite', 2)") + .unwrap() + .done() + .unwrap(); + db.run("insert into u values (1, 'kite')") + .unwrap() + .done() + .unwrap(); + + // Each query keeps strings in an operator: grouping, distinct, sort and join keys, joined + // rows of a hash and a nested-loop join, an aggregate and a window result, an `IN` + // subquery's materialized keys and a set operation's right rows. Every string a query + // returns must still point into the read; strings an operator only compares, such as the + // right rows, are covered by the reruns below. + let queries = [ + "select b from t group by b order by b", + "select distinct b from t order by b", + "select max(b) from t", + "select t.b, u.b from t join u on t.b = u.b order by t.a", + "select max(b) over (partition by c) from t order by a", + "select b from t where b in (select b from u) order by a", + "select b from (select b from t intersect select b from u) s order by b", + "select t.b, u.b from t join u on t.b >= u.b order by t.a, u.a", + ]; + let mut programs = queries + .iter() + .map(|sql| db.program_sql(sql, &[]).unwrap()) + .collect::>(); + let rounds: [(&[&str], [&[&str]; 8]); 2] = [ + ( + &[], + [ + &["db", "kite"], + &["db", "kite"], + &["kite"], + &["kite", "kite"], + &["kite", "kite", "kite"], + &["kite", "kite"], + &["kite"], + &["kite", "kite"], + ], + ), + ( + &[ + "insert into t values (4, 'lmdb', 2)", + "update t set b = 'sql' where a = 2", + "insert into u values (2, 'lmdb')", + ], + [ + &["kite", "lmdb", "sql"], + &["kite", "lmdb", "sql"], + &["sql"], + &["kite", "kite", "lmdb"], + &["sql", "sql", "lmdb", "lmdb"], + &["kite", "kite", "lmdb"], + &["kite", "lmdb"], + &["kite", "sql", "sql", "kite", "lmdb", "lmdb"], + ], + ), + ]; + + // Every round runs the same programs in a new read-only transaction after writes, so a + // string kept from the previous execution would be read after its transaction ended, + // which debug builds catch. + for (writes, expected) in rounds { + for write in writes { + db.run(write).unwrap().done().unwrap(); + } + let mut tx = db.new_transaction(true).unwrap(); + for ((program, sql), expected) in programs.iter_mut().zip(queries).zip(expected) { + let mut iter = program.execute(&mut tx, []).unwrap(); + let mut rows = Vec::new(); + while let Some(row) = iter + .next_tuple(|_, row| { + for value in row.iter() { + let DataValue::Utf8 { value, .. } = value else { + panic!("{sql}: expected a string, got {value:?}") + }; + assert!(value.is_mapped(), "{sql}: {value} was copied"); + } + row[0].to_string() + }) + .unwrap() + { + rows.push(row); + } + iter.done().unwrap(); + assert_eq!(rows, expected, "{sql}"); + } + tx.commit().unwrap(); + } + } + #[test] fn lmdb_backend_smoke() { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); diff --git a/src/types/mapped.rs b/src/types/mapped.rs index 6fa5f276..011f7ac5 100644 --- a/src/types/mapped.rs +++ b/src/types/mapped.rs @@ -115,7 +115,8 @@ pub type BinaryBytes = MaybeMapped<[u8]>; /// A mapped value is only valid until its transaction ends, so it never leaves the engine as is: /// cloning one copies it into an owned value, a result row hands its values out only through /// copying methods, and an executor must not read a value kept from an earlier execution (see -/// `ExecutorNode::reset`). +/// `ExecutorNode::reset`). An executor that keeps a value while it runs, such as a grouping key, +/// copies it with `MaybeMapped::clone_in_txn`, which leaves a mapped value mapped. pub struct MaybeMapped(Repr); enum Repr { @@ -193,6 +194,13 @@ impl MaybeMapped { } } + pub(crate) unsafe fn clone_in_txn(&self) -> Self { + MaybeMapped(match &self.0 { + Repr::Owned(_) => Repr::Owned((**self).to_owned()), + Repr::Mapped(mapped) => Repr::Mapped(mapped.clone()), + }) + } + pub(crate) unsafe fn clone_with(&self, copy: impl FnOnce(&[u8]) -> Vec) -> Self { MaybeMapped(match &self.0 { // SAFETY: the caller guarantees `copy` returns the bytes of a `B`. diff --git a/src/types/value.rs b/src/types/value.rs index 43c8a8fd..2202fee4 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -28,6 +28,7 @@ use chrono::{ use ordered_float::OrderedFloat; #[cfg(feature = "decimal")] use rust_decimal::Decimal; +use std::borrow::Cow; use std::cmp::Ordering; use std::fmt::Formatter; use std::hash::Hash; @@ -978,6 +979,38 @@ impl DataValue { } } + pub(crate) unsafe fn clone_in_txn(&self) -> DataValue { + match self { + DataValue::Utf8 { value, ty, unit } => DataValue::Utf8 { + // SAFETY: the caller's guarantee covers the copy. + value: unsafe { value.clone_in_txn() }, + ty: ty.clone(), + unit: *unit, + }, + DataValue::Binary { value, len } => DataValue::Binary { + // SAFETY: as above. + value: unsafe { value.clone_in_txn() }, + len: *len, + }, + DataValue::Tuple(values) => DataValue::Tuple( + values + .iter() + // SAFETY: as above. + .map(|value| unsafe { value.clone_in_txn() }) + .collect(), + ), + value => value.clone(), + } + } + + pub(crate) unsafe fn own_in_txn(value: Cow<'_, DataValue>) -> DataValue { + match value { + Cow::Owned(value) => value, + // SAFETY: the caller's guarantee covers the copy. + Cow::Borrowed(value) => unsafe { value.clone_in_txn() }, + } + } + #[inline] pub fn is_null(&self) -> bool { matches!(self, DataValue::Null) From 336d60cf8c2bb7595cd4bf8e5d36827cd261cba8 Mon Sep 17 00:00:00 2001 From: kould Date: Thu, 5 Nov 2026 19:33:21 +0800 Subject: [PATCH 10/14] docs(tpcc): refresh 720s TPC-C results on 706f249d --- README.md | 6 +++--- tpcc/README.md | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0515e18c..14aa4481 100755 --- a/README.md +++ b/README.md @@ -135,9 +135,9 @@ print(list(db.run("select * from demo"))) | Backend | TpmC | New-Order | Payment | Order-Status | Delivery | Stock-Level | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| KiteSQL LMDB | 214271 | 252 | 46 | 26 | 340 | 192 | -| KiteSQL RocksDB | 47415 | 360 | 79 | 144 | 15047 | 548 | -| SQLite (WAL, `synchronous=OFF`) | 185982 | 278 | 41 | 34 | 350 | 270 | +| KiteSQL LMDB | 225720 | 240 | 44 | 25 | 326 | 179 | +| KiteSQL RocksDB | 48746 | 343 | 78 | 140 | 14351 | 587 | +| SQLite (WAL, `synchronous=OFF`) | 192110 | 270 | 40 | 33 | 342 | 231 | 👉 [Details and how to reproduce](tpcc/README.md) diff --git a/tpcc/README.md b/tpcc/README.md index d6df38a7..d9a7d336 100644 --- a/tpcc/README.md +++ b/tpcc/README.md @@ -72,11 +72,11 @@ Local 720-second comparison on the machine above: | Backend | TpmC | New-Order p90 (µs) | Payment p90 (µs) | Order-Status p90 (µs) | Delivery p90 (µs) | Stock-Level p90 (µs) | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| KiteSQL LMDB | 214271 | 252 | 46 | 26 | 340 | 192 | -| KiteSQL RocksDB | 47415 | 360 | 79 | 144 | 15047 | 548 | -| SQLite | 185982 | 278 | 41 | 34 | 350 | 270 | +| KiteSQL LMDB | 225720 | 240 | 44 | 25 | 326 | 179 | +| KiteSQL RocksDB | 48746 | 343 | 78 | 140 | 14351 | 587 | +| SQLite | 192110 | 270 | 40 | 33 | 342 | 231 | -- Run date: `2026-11-04`. Latency is measured in microseconds and includes commit. +- Run date: `2026-11-05`. Latency is measured in microseconds and includes commit. - Every row is pinned to `cpu8`, the highest-frequency P-core on this machine (5.4 GHz). - All rows use `--num-ware 1`, `--max-retry 5`, and TPCC's default 720-second measure time. - All rows run without `--sync`, so commits are not fsynced; SQLite uses WAL with `synchronous = OFF`. From 2a0e01a1c32ea63bf229d12b1a17143c41c4a7a1 Mon Sep 17 00:00:00 2001 From: kould Date: Thu, 5 Nov 2026 19:33:21 +0800 Subject: [PATCH 11/14] examples: trace allocations and borrowed parsing against SQLite alloc_trace runs one prepared, index-driven query on KiteSQL (LMDB) and on SQLite, with and without mmap, for notes from 64 B to 256 KiB. It reports latency, malloc and lookaside counts, live heap, and where the returned note pointers point. A multi-op query passes the same rows through grouping, a window and a sort. borrowed_parse keeps a whole result set after the query, as raw rows and parsed from JSON and CSV, borrowing every string on KiteSQL and owning them on SQLite. Both count allocations by interposing glibc malloc, so they run on Linux with glibc only. --- Cargo.lock | 2 + Cargo.toml | 10 + examples/alloc_trace.rs | 697 +++++++++++++++++++++++++++++++++++ examples/borrowed_parse.rs | 542 +++++++++++++++++++++++++++ examples/support/counting.rs | 95 +++++ examples/support/sqlite3.rs | 86 +++++ 6 files changed, 1432 insertions(+) create mode 100644 examples/alloc_trace.rs create mode 100644 examples/borrowed_parse.rs create mode 100644 examples/support/counting.rs create mode 100644 examples/support/sqlite3.rs diff --git a/Cargo.lock b/Cargo.lock index 936107d9..4797ae5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1144,6 +1144,8 @@ dependencies = [ "rocksdb", "rust_decimal", "rustyline", + "serde", + "serde_json", "sqlite", "sqlparser", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 104f6099..e5de3f0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,14 @@ path = "benchmarks/query_benchmark.rs" harness = false required-features = ["pprof"] +[[example]] +name = "alloc_trace" +required-features = ["lmdb", "orm", "parser"] + +[[example]] +name = "borrowed_parse" +required-features = ["lmdb", "orm", "parser"] + [dependencies] bumpalo = { version = "3", default-features = false, features = ["collections"] } hashbrown = { version = "0.15", default-features = false } @@ -76,6 +84,8 @@ indicatif = { version = "0.17" } tempfile = { version = "3.10" } # Benchmark sqlite = { version = "0.34" } +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1" } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Optional dependencies for features diff --git a/examples/alloc_trace.rs b/examples/alloc_trace.rs new file mode 100644 index 00000000..f99c3371 --- /dev/null +++ b/examples/alloc_trace.rs @@ -0,0 +1,697 @@ +//! Runs one prepared, index-driven query on KiteSQL (LMDB) and on SQLite, with `note` strings +//! from 64 B to 256 KiB, and compares latency, allocations and memory. A second query (`multi-op`) +//! reads the same rows but passes them through grouping, a window and a sort first, each of which +//! keeps the strings while it runs. +//! +//! ```shell +//! cargo run --release --example alloc_trace --features lmdb,orm +//! ``` +//! +//! What keeps the comparison fair: +//! - Both engines load the same rows with the same index on `(customer_id, status)`, reopen the +//! database, and run the same SQL, prepared once, with the same parameters, inside one read +//! transaction. Every execution returns about 10 rows. +//! - Both hand each string over as a pointer and a length (`&str` / `sqlite3_column_text`), and +//! the caller reads that string. Whatever an engine copies to produce the pointer is its cost. +//! - Allocations are counted at libc `malloc`, which KiteSQL's Rust code, LMDB and SQLite all +//! call. SQLite serves small allocations from its lookaside slab, which it allocates when the +//! connection opens, without calling `malloc`; those come from `SQLITE_DBSTATUS_LOOKASIDE_HIT` +//! and have their own column. +//! - Memory is the live heap, measured the same way for both: after opening, held by one +//! prepared statement after its first execution, and after the measured executions. SQLite's +//! lookaside slab and page cache are heap and count. LMDB reads pages through a file mapping +//! that the OS page cache backs, so they do not count; but the LMDB environment preallocates +//! page lists for write transactions when it opens, which do. `of which` shows each engine's +//! share: the page cache for SQLite, the LMDB environment alone for KiteSQL. +//! - SQLite also runs with `PRAGMA mmap_size`, which reads pages through a memory map like LMDB. +//! - `notes in file map` checks where each engine's `note` pointer points, from +//! `/proc/self/maps`: into the mapping of the database file, or elsewhere. SQLite copies even +//! a string that fits in its page into a buffer the statement reuses, so it allocates nothing +//! for short strings but still copies them. +//! - Neither engine checks again that a stored string is UTF-8: both encoded it from text. +//! - `prepare+1st` prepares from SQL text on both engines. Most of KiteSQL's share is parsing: +//! KiteSQL parses with `sqlparser`, which supports far more syntax than SQLite's own parser +//! but runs far slower. Programs built with the ORM builder (`Database::program`) skip parsing +//! entirely. +//! +//! Linux with glibc only: counting replaces `malloc` and forwards to glibc's `__libc_malloc`. + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +fn main() -> Result<(), kite_sql::errors::DatabaseError> { + bench::main() +} + +#[cfg(not(all(target_os = "linux", target_env = "gnu")))] +fn main() { + eprintln!("alloc_trace counts allocations through glibc, so it runs on Linux only"); +} + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +#[path = "support/counting.rs"] +mod counting; + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +#[path = "support/sqlite3.rs"] +mod sqlite3; + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +mod bench { + use crate::counting; + use crate::sqlite3::*; + use kite_sql::db::{CatalogKind, DataBaseBuilder, Param, Program, Transaction, TxnResultIter}; + use kite_sql::errors::DatabaseError; + use kite_sql::storage::lmdb::LmdbStorage; + use kite_sql::types::value::DataValue; + use kite_sql::types::{CharLengthUnits, LogicalType}; + use kite_sql::{Model, Projection}; + use std::borrow::Cow; + use std::ffi::{c_int, c_void, CStr, CString}; + use std::path::Path; + use std::ptr; + use std::time::Instant; + + #[derive(Default, Debug, Model)] + #[model(table = "orders")] + #[model(index(name = "orders_customer_status", columns = "customer_id, status"))] + struct Order<'a> { + #[model(primary_key)] + id: i32, + customer_id: i32, + status: Cow<'a, str>, + amount: i32, + note: Cow<'a, str>, + } + + /// A result row whose strings borrow the read transaction when they can. + #[derive(Debug, Projection)] + struct OrderView<'a> { + id: i32, + status: Cow<'a, str>, + amount: i32, + note: Cow<'a, str>, + } + + // `$n` is a numbered parameter in KiteSQL and, numbered by first use, in SQLite. + const QUERY: &str = "select id, status, amount, note from orders \ + where customer_id = $1 and status = $2 and amount > $3"; + /// The same rows as `QUERY`, grouped, ranked by a window and sorted before they are returned. + const MULTI_OP_QUERY: &str = "select status, note, peak, max(peak) over (partition by status) \ + from (select status, note, max(amount) as peak from orders \ + where customer_id = $1 and status = $2 and amount > $3 \ + group by status, note) t \ + order by note desc"; + const STATUSES: [&str; 5] = ["new", "paid", "shipped", "done", "void"]; + /// Every customer has 10 orders per status, so every execution returns about 10 rows. + const ORDERS_PER_CUSTOMER: usize = 10 * STATUSES.len(); + const NOTE_LENS: [usize; 6] = [64, 1 << 10, 4 << 10, 16 << 10, 64 << 10, 256 << 10]; + /// Roughly the note bytes loaded per note length, and read per variant. + const LOAD_BYTES: usize = 64 << 20; + const READ_BYTES: usize = 1 << 30; + /// Fresh statements prepared and run once, for `prepare+1st`. + const FIRST_SAMPLES: usize = 200; + /// Executions before measuring, to warm caches and grow reused buffers. + const WARMUP: usize = 200; + + type Args = (i32, &'static str, i32); + + struct Workload { + note_len: usize, + customers: usize, + runs: usize, + } + + impl Workload { + fn new(note_len: usize) -> Self { + Self { + note_len, + customers: (LOAD_BYTES / (note_len * ORDERS_PER_CUSTOMER)).clamp(5, 400), + runs: (READ_BYTES / (note_len * 10)).clamp(500, 20_000), + } + } + + fn orders(&self) -> usize { + self.customers * ORDERS_PER_CUSTOMER + } + + /// `(customer_id, status, amount, note)` of order `id`. + fn order(&self, id: usize) -> (i32, &'static str, i32, String) { + let mut note = format!("note-{id:06}-"); + note.push_str(&"n".repeat(self.note_len.saturating_sub(note.len()))); + ( + (id % self.customers) as i32, + STATUSES[id / self.customers % STATUSES.len()], + (id % 997) as i32, + note, + ) + } + + /// Parameters of execution `i`: a customer, a status and a minimum amount. + fn args(&self, i: usize) -> Args { + ( + (i % self.customers) as i32, + STATUSES[i % STATUSES.len()], + 10, + ) + } + } + + /// What a variant's executions read. Every variant must read the same rows and text bytes. + #[derive(Default, Clone, Copy, Debug)] + struct Totals { + rows: usize, + text_bytes: usize, + /// Notes that borrowed the transaction, counted by variants that decode into `OrderView`. + borrowed: usize, + } + + /// One variant's measured executions. + struct Run { + label: &'static str, + /// Median of preparing a fresh statement and running it once, for variants that prepare. + first_ns: Option, + ns: Vec, + totals: Totals, + mallocs: usize, + malloc_bytes: usize, + lookaside: Option, + /// Whether `totals.borrowed` was counted. + decodes_views: bool, + } + + /// Live heap of one engine, in bytes. + struct Memory { + label: &'static str, + open: isize, + statement: isize, + after_runs: isize, + /// The part of `after_runs` that the engine's storage holds by design. + of_which: (&'static str, isize), + /// Share of returned `note` pointers inside a mapping of the database file. + notes_in_mapping: f64, + } + + fn warm( + exec: &mut impl FnMut(usize, &mut Totals) -> Result<(), DatabaseError>, + ) -> Result<(), DatabaseError> { + let mut totals = Totals::default(); + (0..WARMUP).try_for_each(|i| exec(i, &mut totals)) + } + + /// Times every execution into `ns`, allocated beforehand, and counts the allocations of all + /// executions together. + fn measure( + label: &'static str, + runs: usize, + mut ns: Vec, + exec: &mut impl FnMut(usize, &mut Totals) -> Result<(), DatabaseError>, + ) -> Result { + assert!(ns.capacity() >= runs); + let mut totals = Totals::default(); + counting::start(); + for i in 0..runs { + let start = Instant::now(); + let result = exec(i, &mut totals); + ns.push(start.elapsed().as_nanos() as u64); + if let Err(err) = result { + counting::stop(); + return Err(err); + } + } + let (mallocs, malloc_bytes) = counting::stop(); + Ok(Run { + label, + first_ns: None, + ns, + totals, + mallocs, + malloc_bytes, + lookaside: None, + decodes_views: false, + }) + } + + fn median(samples: &mut [u64]) -> u64 { + samples.sort_unstable(); + samples[samples.len() / 2] + } + + fn timing_buffers(w: &Workload, count: usize) -> Vec> { + (0..count).map(|_| Vec::with_capacity(w.runs)).collect() + } + + /// Share of `ptrs` that point into a mapping of `file`, from `/proc/self/maps`. + fn share_in_mapping(file: &Path, ptrs: &[usize]) -> f64 { + let maps = std::fs::read_to_string("/proc/self/maps").unwrap(); + let file = file.to_str().unwrap(); + let ranges: Vec<(usize, usize)> = maps + .lines() + .filter(|line| line.ends_with(file)) + .filter_map(|line| { + let (start, end) = line.split_whitespace().next()?.split_once('-')?; + let parse = |hex| usize::from_str_radix(hex, 16).ok(); + Some((parse(start)?, parse(end)?)) + }) + .collect(); + let inside = ptrs + .iter() + .filter(|&&ptr| { + ranges + .iter() + .any(|&(start, end)| (start..end).contains(&ptr)) + }) + .count(); + inside as f64 / ptrs.len() as f64 + } + + fn param_types() -> [(usize, LogicalType); 3] { + [ + (1, LogicalType::Integer), + (2, LogicalType::Varchar(None, CharLengthUnits::Characters)), + (3, LogicalType::Integer), + ] + } + + fn kitesql_params((customer_id, status, amount): Args) -> [(usize, Param<'static>); 3] { + [ + (1, DataValue::Int32(customer_id).into()), + (2, Param::Str(status)), + (3, DataValue::Int32(amount).into()), + ] + } + + /// Reads every string of every row as the `&str` the callback lends. + fn kitesql_read<'p, 'txn: 'p>( + program: &mut Program<'p>, + tx: &mut Transaction<'txn, LmdbStorage>, + args: Args, + totals: &mut Totals, + ) -> Result<(), DatabaseError> { + let mut iter = program.execute(tx, kitesql_params(args))?; + while iter + .next_tuple(|_, row| { + for value in row.iter() { + if let DataValue::Utf8 { value, .. } = value { + let text: &str = value; + totals.text_bytes += text.len(); + } + } + })? + .is_some() + { + totals.rows += 1; + } + Ok(()) + } + + /// Where the `note` of every row returned points. + fn kitesql_note_ptrs<'p, 'txn: 'p>( + program: &mut Program<'p>, + tx: &mut Transaction<'txn, LmdbStorage>, + args: Args, + ) -> Result, DatabaseError> { + let mut ptrs = Vec::new(); + let mut iter = program.execute(tx, kitesql_params(args))?; + while iter + .next_tuple(|_, row| { + if let Some(DataValue::Utf8 { value, .. }) = row.iter().nth(3) { + let text: &str = value; + ptrs.push(text.as_ptr() as usize); + } + })? + .is_some() + {} + Ok(ptrs) + } + + /// Decodes every row into an `OrderView`, whose strings borrow the transaction when they can. + fn kitesql_views<'p, 'txn: 'p>( + program: &mut Program<'p>, + tx: &mut Transaction<'txn, LmdbStorage>, + args: Args, + totals: &mut Totals, + ) -> Result<(), DatabaseError> { + for view in program + .execute(tx, kitesql_params(args))? + .txn_orm::>() + { + let view = view?; + std::hint::black_box((view.id, view.amount)); + totals.rows += 1; + totals.text_bytes += view.status.len() + view.note.len(); + totals.borrowed += usize::from(matches!(view.note, Cow::Borrowed(_))); + } + Ok(()) + } + + fn kitesql( + dir: &Path, + w: &Workload, + lmdb_env: isize, + ) -> Result<(Vec, Memory), DatabaseError> { + { + let mut db = DataBaseBuilder::path(dir).sync(false).build_lmdb()?; + db.create_table::()?; + db.insert_many((0..w.orders()).map(|id| { + let (customer_id, status, amount, note) = w.order(id); + Order { + id: id as i32, + customer_id, + status: status.into(), + amount, + note: note.into(), + } + }))?; + db.analyze_model::()?; + } + + // Allocated before opening, so the live heap measured below holds only the engine's. + let mut first = Vec::with_capacity(FIRST_SAMPLES); + let mut buffers = timing_buffers(w, 4); + let mut runs = Vec::with_capacity(4); + + let before = counting::live(); + let mut db = DataBaseBuilder::path(dir).build_lmdb()?; + let open = counting::live() - before; + // A reopened database loads table catalogs on request; SQLite loads its schema on the + // first prepare. + db.load(CatalogKind::Table("orders".to_string()))?; + let types = param_types(); + + let mut tx = db.new_transaction(true)?; + // Mostly `sqlparser` parsing `QUERY`, kept for its broad SQL syntax; the ORM builder + // (`Database::program`) builds the same program without parsing. + for i in 0..FIRST_SAMPLES { + let start = Instant::now(); + let mut program = tx.program_sql(QUERY, &types)?; + kitesql_read(&mut program, &mut tx, w.args(i), &mut Totals::default())?; + first.push(start.elapsed().as_nanos() as u64); + } + let before_statement = counting::live(); + let mut program = tx.program_sql(QUERY, &types)?; + kitesql_read(&mut program, &mut tx, w.args(0), &mut Totals::default())?; + let statement = counting::live() - before_statement; + let note_ptrs = kitesql_note_ptrs(&mut program, &mut tx, w.args(0))?; + let notes_in_mapping = share_in_mapping(&dir.join("data.mdb"), ¬e_ptrs); + drop(note_ptrs); + + let mut read = + |i, totals: &mut Totals| kitesql_read(&mut program, &mut tx, w.args(i), totals); + warm(&mut read)?; + let ns = buffers.pop().unwrap(); + let mut run = measure("KiteSQL next_tuple", w.runs, ns, &mut read)?; + run.first_ns = Some(median(&mut first)); + runs.push(run); + + let mut views = + |i, totals: &mut Totals| kitesql_views(&mut program, &mut tx, w.args(i), totals); + warm(&mut views)?; + let ns = buffers.pop().unwrap(); + let mut run = measure("KiteSQL txn_orm", w.runs, ns, &mut views)?; + run.decodes_views = true; + runs.push(run); + let after_runs = counting::live() - before; + + let mut multi_op = tx.program_sql(MULTI_OP_QUERY, &types)?; + let mut read = + |i, totals: &mut Totals| kitesql_read(&mut multi_op, &mut tx, w.args(i), totals); + warm(&mut read)?; + let ns = buffers.pop().unwrap(); + runs.push(measure("KiteSQL multi-op", w.runs, ns, &mut read)?); + tx.commit()?; + + // A write transaction is not zero-copy: its pages can move before it ends, so strings are + // copied out. + let mut tx = db.new_transaction(false)?; + let mut views = + |i, totals: &mut Totals| kitesql_views(&mut program, &mut tx, w.args(i), totals); + warm(&mut views)?; + let ns = buffers.pop().unwrap(); + let mut run = measure("KiteSQL txn_orm, write txn", w.runs, ns, &mut views)?; + run.decodes_views = true; + runs.push(run); + tx.commit()?; + + let memory = Memory { + label: "KiteSQL", + open, + statement, + after_runs, + of_which: ("LMDB env", lmdb_env), + notes_in_mapping, + }; + Ok((runs, memory)) + } + + unsafe fn sqlite_load(path: &CStr, w: &Workload) { + let mut db = ptr::null_mut(); + assert_eq!(sqlite3_open(path.as_ptr(), &mut db), 0); + sqlite_exec( + db, + "create table orders(id integer primary key, customer_id int, status text, \ + amount int, note text); \ + create index orders_customer_status on orders(customer_id, status); \ + begin", + ); + let insert = sqlite_prepare(db, c"insert into orders values (?, ?, ?, ?, ?)"); + for id in 0..w.orders() { + let (customer_id, status, amount, note) = w.order(id); + sqlite3_reset(insert); + sqlite3_bind_int64(insert, 1, id as i64); + sqlite3_bind_int64(insert, 2, customer_id as i64); + sqlite3_bind_text( + insert, + 3, + status.as_ptr().cast(), + status.len() as c_int, + None, + ); + sqlite3_bind_int64(insert, 4, amount as i64); + sqlite3_bind_text(insert, 5, note.as_ptr().cast(), note.len() as c_int, None); + assert_eq!(sqlite3_step(insert), DONE); + } + sqlite3_finalize(insert); + sqlite_exec(db, "commit; analyze"); + sqlite3_close(db); + } + + unsafe fn sqlite_bind(stmt: *mut c_void, (customer_id, status, amount): Args) { + sqlite3_reset(stmt); + sqlite3_bind_int64(stmt, 1, customer_id as i64); + // A null destructor is `SQLITE_STATIC`: SQLite borrows the text. + sqlite3_bind_text(stmt, 2, status.as_ptr().cast(), status.len() as c_int, None); + sqlite3_bind_int64(stmt, 3, amount as i64); + } + + /// Where the `note` of every row returned points. + unsafe fn sqlite_note_ptrs(stmt: *mut c_void, args: Args) -> Vec { + sqlite_bind(stmt, args); + let mut ptrs = Vec::new(); + while sqlite3_step(stmt) == ROW { + ptrs.push(sqlite3_column_text(stmt, 3) as usize); + } + ptrs + } + + /// Reads the strings in `columns` of every row through the pointer `sqlite3_column_text` + /// lends. + unsafe fn sqlite_read(stmt: *mut c_void, columns: &[c_int], args: Args, totals: &mut Totals) { + sqlite_bind(stmt, args); + loop { + match sqlite3_step(stmt) { + ROW => { + totals.rows += 1; + for &column in columns { + let text = sqlite3_column_text(stmt, column); + let len = sqlite3_column_bytes(stmt, column) as usize; + let text = + std::str::from_utf8_unchecked(std::slice::from_raw_parts(text, len)); + totals.text_bytes += text.len(); + } + } + DONE => break, + code => panic!("sqlite3_step failed: {code}"), + } + } + } + + unsafe fn sqlite(path: &CStr, w: &Workload, mmap: bool) -> (Vec, Memory) { + let (label, multi_op_label) = if mmap { + ("SQLite mmap_size", "SQLite mmap_size, multi-op") + } else { + ("SQLite", "SQLite multi-op") + }; + // Allocated before opening, so the live heap measured below holds only the engine's. + let mut first = Vec::with_capacity(FIRST_SAMPLES); + let mut buffers = timing_buffers(w, 2); + let query = CString::new(QUERY).unwrap(); + let multi_op_query = CString::new(MULTI_OP_QUERY).unwrap(); + + let before = counting::live(); + let mut db = ptr::null_mut(); + assert_eq!(sqlite3_open(path.as_ptr(), &mut db), 0); + let open = counting::live() - before; + if mmap { + sqlite_exec(db, "pragma mmap_size = 1073741824"); + } + + // One read transaction for every execution, as KiteSQL has. + sqlite_exec(db, "begin"); + for i in 0..FIRST_SAMPLES { + let start = Instant::now(); + let stmt = sqlite_prepare(db, &query); + sqlite_read(stmt, &[1, 3], w.args(i), &mut Totals::default()); + first.push(start.elapsed().as_nanos() as u64); + sqlite3_finalize(stmt); + } + let stmt = sqlite_prepare(db, &query); + sqlite_read(stmt, &[1, 3], w.args(0), &mut Totals::default()); + let statement = sqlite3_stmt_status(stmt, STMTSTATUS_MEMUSED, 0) as isize; + let note_ptrs = sqlite_note_ptrs(stmt, w.args(0)); + let notes_in_mapping = share_in_mapping(Path::new(path.to_str().unwrap()), ¬e_ptrs); + drop(note_ptrs); + + let mut runs = Vec::with_capacity(2); + let mut read = |i, totals: &mut Totals| { + sqlite_read(stmt, &[1, 3], w.args(i), totals); + Ok(()) + }; + warm(&mut read).unwrap(); + db_status(db, DBSTATUS_LOOKASIDE_HIT); + let ns = buffers.pop().unwrap(); + let mut run = measure(label, w.runs, ns, &mut read).unwrap(); + run.lookaside = Some(db_status(db, DBSTATUS_LOOKASIDE_HIT).1 as usize); + run.first_ns = Some(median(&mut first)); + runs.push(run); + let after_runs = counting::live() - before; + let page_cache = db_status(db, DBSTATUS_CACHE_USED).0 as isize; + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + + let stmt = sqlite_prepare(db, &multi_op_query); + let mut read = |i, totals: &mut Totals| { + sqlite_read(stmt, &[0, 1], w.args(i), totals); + Ok(()) + }; + warm(&mut read).unwrap(); + db_status(db, DBSTATUS_LOOKASIDE_HIT); + let ns = buffers.pop().unwrap(); + let mut run = measure(multi_op_label, w.runs, ns, &mut read).unwrap(); + run.lookaside = Some(db_status(db, DBSTATUS_LOOKASIDE_HIT).1 as usize); + runs.push(run); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + sqlite_exec(db, "commit"); + sqlite3_close(db); + let memory = Memory { + label, + open, + statement, + after_runs, + of_which: ("page cache", page_cache), + notes_in_mapping, + }; + (runs, memory) + } + + /// Heap the LMDB environment holds after opening, outside KiteSQL. + fn lmdb_env_bytes(dir: &Path) -> Result { + let before = counting::live(); + let env = lmdb::Environment::new() + .set_flags(lmdb::EnvironmentFlags::NO_TLS) + .open(dir)?; + let bytes = counting::live() - before; + drop(env); + Ok(bytes) + } + + fn print(w: &Workload, runs: &mut [Run], memory: &[Memory]) { + let rows = runs[0].totals.rows as f64 / w.runs as f64; + println!( + "\nnote {} B: {} orders; {} executions per variant, {rows:.1} rows each", + w.note_len, + w.orders(), + w.runs + ); + println!( + "{:<28} | {:>16} | {:>8} | {:>8} | {:>11} | {:>14} | {:>15} | {:>14}", + "variant", + "prepare+1st µs", + "p50 µs", + "p99 µs", + "malloc/exec", + "lookaside/exec", + "malloc KiB/exec", + "borrowed notes" + ); + let per_exec = |count: usize| count as f64 / w.runs as f64; + for run in runs.iter_mut() { + let first = run + .first_ns + .map_or("-".to_string(), |ns| format!("{:.2}", ns as f64 / 1e3)); + let p99_index = (run.ns.len() - 1) * 99 / 100; + let p50 = median(&mut run.ns) as f64 / 1e3; + let p99 = run.ns[p99_index] as f64 / 1e3; + let lookaside = run + .lookaside + .map_or("-".to_string(), |hits| format!("{:.2}", per_exec(hits))); + let borrowed = if run.decodes_views { + format!( + "{:.0}%", + run.totals.borrowed as f64 * 100.0 / run.totals.rows as f64 + ) + } else { + "-".to_string() + }; + println!( + "{:<28} | {first:>16} | {p50:>8.2} | {p99:>8.2} | {:>11.2} | {lookaside:>14} | {:>15.2} | {borrowed:>14}", + run.label, + per_exec(run.mallocs), + per_exec(run.malloc_bytes) / 1024.0, + ); + } + let kib = |bytes: isize| format!("{:.1}", bytes as f64 / 1024.0); + println!( + "{:<28} | {:>16} | {:>14} | {:>16} | {:>20} | {:>17}", + "live heap (KiB)", "open", "statement", "after runs", "of which", "notes in file map" + ); + for memory in memory { + println!( + "{:<28} | {:>16} | {:>14} | {:>16} | {:>20} | {:>17}", + memory.label, + kib(memory.open), + kib(memory.statement), + kib(memory.after_runs), + format!("{} {}", kib(memory.of_which.1), memory.of_which.0), + format!("{:.0}%", memory.notes_in_mapping * 100.0), + ); + } + } + + pub fn main() -> Result<(), DatabaseError> { + let version = unsafe { CStr::from_ptr(sqlite3_libversion()) }; + let scratch = tempfile::TempDir::new()?; + let lmdb_env = lmdb_env_bytes(scratch.path())?; + println!("SQLite {}", version.to_string_lossy()); + for note_len in NOTE_LENS { + let w = Workload::new(note_len); + let dir = tempfile::TempDir::new()?; + let (mut runs, kitesql_memory) = kitesql(&dir.path().join("kitesql"), &w, lmdb_env)?; + let mut memory = vec![kitesql_memory]; + let path = dir.path().join("orders.sqlite"); + let path = CString::new(path.to_str().unwrap()).unwrap(); + unsafe { sqlite_load(&path, &w) }; + for mmap in [false, true] { + let (sqlite_runs, sqlite_memory) = unsafe { sqlite(&path, &w, mmap) }; + runs.extend(sqlite_runs); + memory.push(sqlite_memory); + } + // `multi-op` returns the same rows and strings as `QUERY`, in another order. + let read = |run: &Run| (run.totals.rows, run.totals.text_bytes); + for run in &runs { + assert_eq!(read(run), read(&runs[0]), "{} read other rows", run.label); + } + print(&w, &mut runs, &memory); + } + Ok(()) + } +} diff --git a/examples/borrowed_parse.rs b/examples/borrowed_parse.rs new file mode 100644 index 00000000..8b7c3611 --- /dev/null +++ b/examples/borrowed_parse.rs @@ -0,0 +1,542 @@ +//! Collects a whole result set and keeps it after the query: as raw rows, parsed from JSON, and +//! parsed from CSV. KiteSQL (LMDB) borrows every string from its read transaction; SQLite hands +//! a string out only until the next `sqlite3_step`, so keeping one means owning it. +//! +//! ```shell +//! cargo run --release --example borrowed_parse --features lmdb,orm +//! ``` +//! +//! Every execution selects one group of 100 documents by an index on `grp`, and builds a `Vec` of +//! records that outlives the loop over the rows; afterwards their fields are summed, and both +//! engines must arrive at the same sums. +//! - `collect`: `(id, body)` pairs. KiteSQL keeps `Cow::Borrowed` bodies; SQLite copies each +//! body into a `String`. +//! - `json`: `serde_json` into a struct of `&str` fields borrowed from the stored text on +//! KiteSQL, and of `String` fields on SQLite. +//! - `csv`: the same fields split from a comma-separated line, without quoting. +//! +//! SQLite runs the common way: records own their strings. Its cheapest alternative copies each +//! row's text once into a `String` and borrows the fields from that, one allocation per row +//! instead of one per field. +//! +//! SQLite reads pages through `PRAGMA mmap_size`. Allocations are counted at libc `malloc`, as in +//! `alloc_trace`, with SQLite's lookaside allocations in their own column. +//! +//! Linux with glibc only: counting replaces `malloc` and forwards to glibc's `__libc_malloc`. + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +fn main() -> Result<(), kite_sql::errors::DatabaseError> { + bench::main() +} + +#[cfg(not(all(target_os = "linux", target_env = "gnu")))] +fn main() { + eprintln!("borrowed_parse counts allocations through glibc, so it runs on Linux only"); +} + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +#[path = "support/counting.rs"] +mod counting; + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +#[path = "support/sqlite3.rs"] +mod sqlite3; + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +mod bench { + use crate::counting; + use crate::sqlite3::*; + use kite_sql::db::{CatalogKind, DataBaseBuilder, Program, Transaction, TxnResultIter}; + use kite_sql::errors::DatabaseError; + use kite_sql::storage::lmdb::LmdbStorage; + use kite_sql::types::value::DataValue; + use kite_sql::types::LogicalType; + use kite_sql::{Model, Projection}; + use serde::Deserialize; + use std::borrow::Cow; + use std::ffi::{c_int, c_void, CStr, CString}; + use std::path::Path; + use std::ptr; + use std::time::Instant; + + #[derive(Default, Debug, Model)] + #[model(table = "json_docs")] + #[model(index(name = "json_docs_grp", columns = "grp"))] + struct JsonDoc<'a> { + #[model(primary_key)] + id: i32, + grp: i32, + body: Cow<'a, str>, + } + + #[derive(Default, Debug, Model)] + #[model(table = "csv_docs")] + #[model(index(name = "csv_docs_grp", columns = "grp"))] + struct CsvDoc<'a> { + #[model(primary_key)] + id: i32, + grp: i32, + body: Cow<'a, str>, + } + + /// A selected row, whose body borrows the read transaction. + #[derive(Debug, Projection)] + struct Raw<'a> { + id: i32, + body: Cow<'a, str>, + } + + /// A parsed document whose fields borrow the text they were parsed from. + #[derive(Deserialize)] + struct Record<'a> { + id: i64, + user: &'a str, + email: &'a str, + title: &'a str, + tags: [&'a str; 3], + body: &'a str, + } + + /// The same document owning its strings, as SQLite has to keep it. + #[derive(Deserialize)] + struct OwnedRecord { + id: i64, + user: String, + email: String, + title: String, + tags: [String; 3], + body: String, + } + + impl Record<'_> { + fn parse_csv(line: &str) -> Record<'_> { + let mut fields = line.splitn(8, ','); + let mut next = || fields.next().expect("eight fields"); + Record { + id: next().parse().expect("numeric id"), + user: next(), + email: next(), + title: next(), + tags: [next(), next(), next()], + body: next(), + } + } + + fn to_owned_record(&self) -> OwnedRecord { + OwnedRecord { + id: self.id, + user: self.user.to_owned(), + email: self.email.to_owned(), + title: self.title.to_owned(), + tags: self.tags.map(str::to_owned), + body: self.body.to_owned(), + } + } + + fn fields(&self) -> (i64, [&str; 7]) { + let [t0, t1, t2] = self.tags; + let fields = [self.user, self.email, self.title, t0, t1, t2, self.body]; + (self.id, fields) + } + } + + impl OwnedRecord { + fn fields(&self) -> (i64, [&str; 7]) { + let [t0, t1, t2] = &self.tags; + let fields = [&self.user, &self.email, &self.title, t0, t1, t2, &self.body]; + (self.id, fields.map(String::as_str)) + } + } + + const PER_GROUP: usize = 100; + const DOC_LENS: [usize; 3] = [256, 2 << 10, 16 << 10]; + /// Roughly the document bytes loaded per table, and read per scenario and engine. + const LOAD_BYTES: usize = 64 << 20; + const READ_BYTES: usize = 1 << 30; + /// Executions before measuring, to warm caches and grow reused buffers. + const WARMUP: usize = 100; + const JSON_QUERY: &str = "select id, body from json_docs where grp = $1"; + const CSV_QUERY: &str = "select id, body from csv_docs where grp = $1"; + + struct Workload { + doc_len: usize, + groups: usize, + runs: usize, + } + + impl Workload { + fn new(doc_len: usize) -> Self { + Self { + doc_len, + groups: (LOAD_BYTES / (doc_len * PER_GROUP)).clamp(4, 200), + runs: (READ_BYTES / (doc_len * PER_GROUP)).clamp(300, 5_000), + } + } + + fn docs(&self) -> usize { + self.groups * PER_GROUP + } + + /// Document `id` as JSON and as CSV, with the same fields, about `doc_len` bytes long. + fn doc(&self, id: usize) -> (String, String) { + let user = format!("user-{id:06}"); + let email = format!("{user}@example.com"); + let title = format!("document number {id}"); + let tags = [id % 7, id % 11, id % 13].map(|n| format!("tag-{n}")); + let [t0, t1, t2] = &tags; + let fill = self.doc_len.saturating_sub(120 + 2 * user.len()); + let body = "b".repeat(fill); + let json = format!( + r#"{{"id":{id},"user":"{user}","email":"{email}","title":"{title}","tags":["{t0}","{t1}","{t2}"],"body":"{body}"}}"# + ); + let csv = format!("{id},{user},{email},{title},{t0},{t1},{t2},{body}"); + (json, csv) + } + + fn group(&self, i: usize) -> i32 { + (i % self.groups) as i32 + } + } + + /// What the kept records add up to. Every scenario on both engines must arrive at the same. + #[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] + struct Totals { + rows: usize, + ids: i64, + bytes: usize, + } + + impl Totals { + fn add(&mut self, id: i64, fields: &[&str]) { + self.rows += 1; + self.ids += id; + self.bytes += fields.iter().map(|field| field.len()).sum::(); + } + } + + /// One engine's measured executions of one scenario. + struct Run { + ns: Vec, + totals: Totals, + mallocs: usize, + malloc_bytes: usize, + lookaside: Option, + } + + type Exec<'a> = dyn FnMut(usize, &mut Totals) -> Result<(), DatabaseError> + 'a; + + fn warm(exec: &mut Exec<'_>) -> Result<(), DatabaseError> { + let mut totals = Totals::default(); + (0..WARMUP).try_for_each(|i| exec(i, &mut totals)) + } + + /// Times every execution, and counts the allocations of all executions together. + fn measure(runs: usize, exec: &mut Exec<'_>) -> Result { + let mut ns = Vec::with_capacity(runs); + let mut totals = Totals::default(); + counting::start(); + for i in 0..runs { + let start = Instant::now(); + let result = exec(i, &mut totals); + ns.push(start.elapsed().as_nanos() as u64); + if let Err(err) = result { + counting::stop(); + return Err(err); + } + } + let (mallocs, malloc_bytes) = counting::stop(); + Ok(Run { + ns, + totals, + mallocs, + malloc_bytes, + lookaside: None, + }) + } + + fn kite_params(grp: i32) -> [(usize, kite_sql::db::Param<'static>); 1] { + [(1, DataValue::Int32(grp).into())] + } + + /// The text a read-only transaction lent. + fn lent(body: Cow<'_, str>) -> &str { + match body { + Cow::Borrowed(text) => text, + Cow::Owned(_) => unreachable!("a read-only LMDB transaction lends its strings"), + } + } + + /// Keeps every row as `(id, body)`, with the body borrowed from the transaction. + fn kite_collect<'p, 'txn: 'p>( + program: &mut Program<'p>, + tx: &mut Transaction<'txn, LmdbStorage>, + grp: i32, + totals: &mut Totals, + ) -> Result<(), DatabaseError> { + let mut rows = Vec::with_capacity(PER_GROUP); + for raw in program.execute(tx, kite_params(grp))?.txn_orm::>() { + rows.push(raw?); + } + for raw in &rows { + totals.add(raw.id as i64, &[&raw.body]); + } + Ok(()) + } + + /// Parses every row into a `Record` whose fields borrow the transaction. + fn kite_parse<'p, 'txn: 'p>( + program: &mut Program<'p>, + tx: &mut Transaction<'txn, LmdbStorage>, + grp: i32, + totals: &mut Totals, + parse: for<'a> fn(&'a str) -> Record<'a>, + ) -> Result<(), DatabaseError> { + let mut records = Vec::with_capacity(PER_GROUP); + for raw in program.execute(tx, kite_params(grp))?.txn_orm::>() { + records.push(parse(lent(raw?.body))); + } + for record in &records { + let (id, fields) = record.fields(); + totals.add(id, &fields); + } + Ok(()) + } + + fn parse_json(text: &str) -> Record<'_> { + serde_json::from_str(text).expect("valid JSON without escapes") + } + + /// Runs `stmt` for group `grp`, lending each row's id and text to `f` until the next step. + unsafe fn sqlite_rows(stmt: *mut c_void, grp: i32, mut f: impl FnMut(i64, &str)) { + sqlite3_reset(stmt); + sqlite3_bind_int64(stmt, 1, grp as i64); + loop { + match sqlite3_step(stmt) { + ROW => { + let id = sqlite3_column_int64(stmt, 0); + let text = sqlite3_column_text(stmt, 1); + let len = sqlite3_column_bytes(stmt, 1) as usize; + f( + id, + std::str::from_utf8_unchecked(std::slice::from_raw_parts(text, len)), + ); + } + DONE => break, + code => panic!("sqlite3_step failed: {code}"), + } + } + } + + /// Keeps every row as `(id, body)`, copying the body out before the next step. + unsafe fn sqlite_collect( + stmt: *mut c_void, + grp: i32, + totals: &mut Totals, + ) -> Result<(), DatabaseError> { + let mut rows = Vec::with_capacity(PER_GROUP); + sqlite_rows(stmt, grp, |id, text| rows.push((id, text.to_owned()))); + for (id, body) in &rows { + totals.add(*id, &[body]); + } + Ok(()) + } + + /// Parses every row into an `OwnedRecord`, the common way to keep it past the next step. + unsafe fn sqlite_parse( + stmt: *mut c_void, + grp: i32, + totals: &mut Totals, + parse: fn(&str) -> OwnedRecord, + ) -> Result<(), DatabaseError> { + let mut records = Vec::with_capacity(PER_GROUP); + sqlite_rows(stmt, grp, |_, text| records.push(parse(text))); + for record in &records { + let (id, fields) = record.fields(); + totals.add(id, &fields); + } + Ok(()) + } + + fn parse_json_owned(text: &str) -> OwnedRecord { + serde_json::from_str(text).expect("valid JSON") + } + + fn parse_csv_owned(text: &str) -> OwnedRecord { + Record::parse_csv(text).to_owned_record() + } + + type Docs = [(String, String)]; + + fn kitesql(dir: &Path, w: &Workload, docs: &Docs) -> Result<[Run; 3], DatabaseError> { + { + let mut db = DataBaseBuilder::path(dir).sync(false).build_lmdb()?; + db.create_table::()?; + db.create_table::()?; + let grp = |id: usize| (id % w.groups) as i32; + db.insert_many(docs.iter().enumerate().map(|(id, (json, _))| JsonDoc { + id: id as i32, + grp: grp(id), + body: json.as_str().into(), + }))?; + db.insert_many(docs.iter().enumerate().map(|(id, (_, csv))| CsvDoc { + id: id as i32, + grp: grp(id), + body: csv.as_str().into(), + }))?; + db.analyze_model::()?; + db.analyze_model::()?; + } + let mut db = DataBaseBuilder::path(dir).build_lmdb()?; + db.load(CatalogKind::Table("json_docs".to_string()))?; + db.load(CatalogKind::Table("csv_docs".to_string()))?; + let types = [(1, LogicalType::Integer)]; + let mut tx = db.new_transaction(true)?; + let mut json = tx.program_sql(JSON_QUERY, &types)?; + let mut csv = tx.program_sql(CSV_QUERY, &types)?; + + let mut collect = + |i: usize, t: &mut Totals| kite_collect(&mut json, &mut tx, w.group(i), t); + warm(&mut collect)?; + let collected = measure(w.runs, &mut collect)?; + let mut parse = + |i: usize, t: &mut Totals| kite_parse(&mut json, &mut tx, w.group(i), t, parse_json); + warm(&mut parse)?; + let parsed_json = measure(w.runs, &mut parse)?; + let mut parse = |i: usize, t: &mut Totals| { + kite_parse(&mut csv, &mut tx, w.group(i), t, Record::parse_csv) + }; + warm(&mut parse)?; + let parsed_csv = measure(w.runs, &mut parse)?; + tx.commit()?; + Ok([collected, parsed_json, parsed_csv]) + } + + unsafe fn sqlite(path: &CStr, w: &Workload, docs: &Docs) -> [Run; 3] { + let mut db = ptr::null_mut(); + assert_eq!(sqlite3_open(path.as_ptr(), &mut db), 0); + sqlite_exec( + db, + "create table json_docs(id integer primary key, grp int, body text); \ + create index json_docs_grp on json_docs(grp); \ + create table csv_docs(id integer primary key, grp int, body text); \ + create index csv_docs_grp on csv_docs(grp); \ + begin", + ); + for (table, csv) in [("json_docs", false), ("csv_docs", true)] { + let sql = CString::new(format!("insert into {table} values (?, ?, ?)")).unwrap(); + let insert = sqlite_prepare(db, &sql); + for (id, (json_body, csv_body)) in docs.iter().enumerate() { + let body = if csv { csv_body } else { json_body }; + sqlite3_reset(insert); + sqlite3_bind_int64(insert, 1, id as i64); + sqlite3_bind_int64(insert, 2, (id % w.groups) as i64); + sqlite3_bind_text(insert, 3, body.as_ptr().cast(), body.len() as c_int, None); + assert_eq!(sqlite3_step(insert), DONE); + } + sqlite3_finalize(insert); + } + sqlite_exec(db, "commit; analyze"); + sqlite3_close(db); + + let mut db = ptr::null_mut(); + assert_eq!(sqlite3_open(path.as_ptr(), &mut db), 0); + sqlite_exec(db, "pragma mmap_size = 1073741824; begin"); + let json = sqlite_prepare(db, &CString::new(JSON_QUERY).unwrap()); + let csv = sqlite_prepare(db, &CString::new(CSV_QUERY).unwrap()); + let run = |exec: &mut Exec<'_>| { + warm(exec).unwrap(); + db_status(db, DBSTATUS_LOOKASIDE_HIT); + let mut run = measure(w.runs, exec).unwrap(); + run.lookaside = Some(db_status(db, DBSTATUS_LOOKASIDE_HIT).1 as usize); + run + }; + let runs = [ + run(&mut |i, t| sqlite_collect(json, w.group(i), t)), + run(&mut |i, t| sqlite_parse(json, w.group(i), t, parse_json_owned)), + run(&mut |i, t| sqlite_parse(csv, w.group(i), t, parse_csv_owned)), + ]; + for stmt in [json, csv] { + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + } + sqlite_exec(db, "commit"); + sqlite3_close(db); + runs + } + + fn print(w: &Workload, kite: &mut [Run; 3], lite: &mut [Run; 3]) { + println!( + "\ndoc {} B: {} docs per table; {} executions of {PER_GROUP} rows per scenario", + w.doc_len, + w.docs(), + w.runs + ); + let columns = |engine: &str| { + format!( + "{engine:>11} | {:>8} | {:>10} | {:>8}", + "parse", "malloc/row", "KiB/exec" + ) + }; + println!( + "{:<8} | {} | {} | {:>13}", + "scenario", + columns("KiteSQL p50"), + columns("SQLite p50"), + "lookaside/row" + ); + let rows = (w.runs * PER_GROUP) as f64; + let p50 = |run: &mut Run| { + run.ns.sort_unstable(); + run.ns[run.ns.len() / 2] as f64 / 1e3 + }; + let collect = [p50(&mut kite[0]), p50(&mut lite[0])]; + for (i, scenario) in ["collect", "json", "csv"].iter().enumerate() { + let mut line = format!("{scenario:<8} |"); + for (run, collect) in [(&mut kite[i], collect[0]), (&mut lite[i], collect[1])] { + let p50 = p50(run); + let parse = if i == 0 { + "-".to_string() + } else { + format!("{:.2}", p50 - collect) + }; + line += &format!( + " {p50:>11.2} | {parse:>8} | {:>10.2} | {:>8.2} |", + run.mallocs as f64 / rows, + run.malloc_bytes as f64 / w.runs as f64 / 1024.0, + ); + } + let hits = lite[i].lookaside.unwrap_or_default(); + println!("{line} {:>13.2}", hits as f64 / rows); + } + } + + pub fn main() -> Result<(), DatabaseError> { + let version = unsafe { CStr::from_ptr(sqlite3_libversion()) }; + println!("SQLite {} with mmap_size", version.to_string_lossy()); + println!( + "parse: p50 minus the same engine's `collect` p50. malloc/row: malloc calls per row, \ + SQLite's lookaside allocations aside." + ); + for doc_len in DOC_LENS { + let w = Workload::new(doc_len); + let docs: Vec<_> = (0..w.docs()).map(|id| w.doc(id)).collect(); + let dir = tempfile::TempDir::new()?; + let mut kite = kitesql(&dir.path().join("kitesql"), &w, &docs)?; + let path = CString::new(dir.path().join("docs.sqlite").to_str().unwrap()).unwrap(); + let mut lite = unsafe { sqlite(&path, &w, &docs) }; + for (kite, lite) in kite.iter().zip(&lite) { + assert_eq!( + kite.totals, lite.totals, + "the engines kept different records" + ); + assert_eq!(kite.totals.rows, w.runs * PER_GROUP); + } + assert_eq!( + kite[1].totals, kite[2].totals, + "JSON and CSV hold the same fields" + ); + print(&w, &mut kite, &mut lite); + } + Ok(()) + } +} diff --git a/examples/support/counting.rs b/examples/support/counting.rs new file mode 100644 index 00000000..7067cbfa --- /dev/null +++ b/examples/support/counting.rs @@ -0,0 +1,95 @@ +//! Counts allocation calls and requested bytes while enabled, and tracks the live heap always. + +use std::ffi::{c_int, c_void}; +use std::sync::atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering::Relaxed}; + +static COUNTING: AtomicBool = AtomicBool::new(false); +static CALLS: AtomicUsize = AtomicUsize::new(0); +static BYTES: AtomicUsize = AtomicUsize::new(0); +static LIVE: AtomicIsize = AtomicIsize::new(0); + +extern "C" { + fn __libc_malloc(size: usize) -> *mut c_void; + fn __libc_calloc(count: usize, size: usize) -> *mut c_void; + fn __libc_realloc(ptr: *mut c_void, size: usize) -> *mut c_void; + fn __libc_memalign(align: usize, size: usize) -> *mut c_void; + fn __libc_free(ptr: *mut c_void); + fn malloc_usable_size(ptr: *mut c_void) -> usize; +} + +/// Bytes currently allocated. +#[allow(dead_code)] // Not every example measures the live heap. +pub fn live() -> isize { + LIVE.load(Relaxed) +} + +pub fn start() { + CALLS.store(0, Relaxed); + BYTES.store(0, Relaxed); + COUNTING.store(true, Relaxed); +} + +/// Stops counting and returns the allocation calls and the bytes they requested since `start`. +pub fn stop() -> (usize, usize) { + COUNTING.store(false, Relaxed); + (CALLS.load(Relaxed), BYTES.load(Relaxed)) +} + +unsafe fn track(ptr: *mut c_void, size: usize) -> *mut c_void { + if COUNTING.load(Relaxed) { + CALLS.fetch_add(1, Relaxed); + BYTES.fetch_add(size, Relaxed); + } + if !ptr.is_null() { + LIVE.fetch_add(malloc_usable_size(ptr) as isize, Relaxed); + } + ptr +} + +unsafe fn untrack(ptr: *mut c_void) { + if !ptr.is_null() { + LIVE.fetch_sub(malloc_usable_size(ptr) as isize, Relaxed); + } +} + +#[no_mangle] +unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + track(__libc_malloc(size), size) +} + +#[no_mangle] +unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void { + track(__libc_calloc(count, size), count.saturating_mul(size)) +} + +#[no_mangle] +unsafe extern "C" fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void { + untrack(ptr); + track(__libc_realloc(ptr, size), size) +} + +#[no_mangle] +unsafe extern "C" fn memalign(align: usize, size: usize) -> *mut c_void { + track(__libc_memalign(align, size), size) +} + +#[no_mangle] +unsafe extern "C" fn aligned_alloc(align: usize, size: usize) -> *mut c_void { + track(__libc_memalign(align, size), size) +} + +#[no_mangle] +unsafe extern "C" fn posix_memalign(out: *mut *mut c_void, align: usize, size: usize) -> c_int { + let ptr = track(__libc_memalign(align, size), size); + if ptr.is_null() { + return 12; // ENOMEM + } + *out = ptr; + 0 +} + +#[no_mangle] +unsafe extern "C" fn free(ptr: *mut c_void) { + untrack(ptr); + __libc_free(ptr) +} diff --git a/examples/support/sqlite3.rs b/examples/support/sqlite3.rs new file mode 100644 index 00000000..85cb2d22 --- /dev/null +++ b/examples/support/sqlite3.rs @@ -0,0 +1,86 @@ +//! The parts of the SQLite C API the examples call, from the system library that the `sqlite` +//! dev-dependency also links. The raw API lets the caller read the string SQLite hands out in +//! place, with no further copy; SQLite itself may still have copied it there. Not every +//! example calls every function. +#![allow(dead_code)] + +use std::ffi::{c_char, c_int, c_void, CStr, CString}; +use std::ptr; + +pub const ROW: c_int = 100; +pub const DONE: c_int = 101; +pub const DBSTATUS_CACHE_USED: c_int = 1; +pub const DBSTATUS_LOOKASIDE_HIT: c_int = 4; +pub const STMTSTATUS_MEMUSED: c_int = 99; + +#[link(name = "sqlite3")] +extern "C" { + pub fn sqlite3_libversion() -> *const c_char; + pub fn sqlite3_open(path: *const c_char, db: *mut *mut c_void) -> c_int; + pub fn sqlite3_close(db: *mut c_void) -> c_int; + pub fn sqlite3_exec( + db: *mut c_void, + sql: *const c_char, + callback: *const c_void, + arg: *mut c_void, + err: *mut *mut c_char, + ) -> c_int; + pub fn sqlite3_prepare_v2( + db: *mut c_void, + sql: *const c_char, + len: c_int, + stmt: *mut *mut c_void, + tail: *mut *const c_char, + ) -> c_int; + pub fn sqlite3_finalize(stmt: *mut c_void) -> c_int; + pub fn sqlite3_reset(stmt: *mut c_void) -> c_int; + pub fn sqlite3_bind_int64(stmt: *mut c_void, index: c_int, value: i64) -> c_int; + pub fn sqlite3_bind_text( + stmt: *mut c_void, + index: c_int, + value: *const c_char, + len: c_int, + destructor: Option, + ) -> c_int; + pub fn sqlite3_step(stmt: *mut c_void) -> c_int; + pub fn sqlite3_column_bytes(stmt: *mut c_void, column: c_int) -> c_int; + pub fn sqlite3_column_int64(stmt: *mut c_void, column: c_int) -> i64; + pub fn sqlite3_column_text(stmt: *mut c_void, column: c_int) -> *const u8; + pub fn sqlite3_db_status( + db: *mut c_void, + op: c_int, + cur: *mut c_int, + hi: *mut c_int, + reset: c_int, + ) -> c_int; + pub fn sqlite3_stmt_status(stmt: *mut c_void, op: c_int, reset: c_int) -> c_int; +} + +pub unsafe fn sqlite_exec(db: *mut c_void, sql: &str) { + let sql = CString::new(sql).unwrap(); + let code = sqlite3_exec( + db, + sql.as_ptr(), + ptr::null(), + ptr::null_mut(), + ptr::null_mut(), + ); + assert_eq!(code, 0, "{sql:?}"); +} + +pub unsafe fn sqlite_prepare(db: *mut c_void, sql: &CStr) -> *mut c_void { + let mut stmt = ptr::null_mut(); + let code = sqlite3_prepare_v2(db, sql.as_ptr(), -1, &mut stmt, ptr::null_mut()); + assert_eq!(code, 0, "{sql:?}"); + stmt +} + +/// Returns a counter's current value and highwater mark, and resets the highwater mark. +pub unsafe fn db_status(db: *mut c_void, op: c_int) -> (c_int, c_int) { + let (mut current, mut highwater) = (0, 0); + assert_eq!( + sqlite3_db_status(db, op, &mut current, &mut highwater, 1), + 0 + ); + (current, highwater) +} From 67eb72c775f4f985d2cf68f28e99c3bdcfbb9fbe Mon Sep 17 00:00:00 2001 From: kould Date: Fri, 6 Nov 2026 19:10:53 +0800 Subject: [PATCH 12/14] refactor(index-scan): hold scan ranges in a Cow IndexScan borrows the plan's static range or owns one that parameters bind into, so the ScanRanges enum and its range() accessor go away. --- src/db/program.rs | 11 ++++----- src/execution/dql/index_scan.rs | 40 ++++++++++++--------------------- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/db/program.rs b/src/db/program.rs index 3fee9e76..fd643469 100644 --- a/src/db/program.rs +++ b/src/db/program.rs @@ -266,7 +266,7 @@ impl<'p, T: StorageTransaction + 'p> ResultIter for ProgramIter<'_, 'p, T> { mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; - use crate::execution::dql::index_scan::{IndexScan, ScanRanges}; + use crate::execution::dql::index_scan::IndexScan; use crate::expression::range_detacher::{Range, RangeValue}; use crate::expression::{BinaryOperator, ScalarExpression}; use crate::planner::operator::filter::FilterOperator; @@ -277,6 +277,7 @@ mod tests { use crate::types::tuple::TupleLike; use crate::types::value::DataValue; use crate::types::LogicalType; + use std::borrow::Cow; use std::ops::Bound; #[test] @@ -462,7 +463,7 @@ mod tests { let mut exec_arena = crate::execution::ExecArena::new(context, &transaction); let mut execution_arena = crate::execution::ExecContext::new(&mut exec_buffer, &mut exec_arena); - let mut ranges = ScanRanges::Owned(Range::Dummy); + let mut ranges = Cow::Owned(Range::Dummy); IndexScan::bind_ranges( info, info.lookup.as_ref().expect("lookup"), @@ -480,7 +481,7 @@ mod tests { } else { original }; - assert_eq!(ranges.range(), want); + assert_eq!(&*ranges, want); } assert_eq!( scan.physical_option.as_ref().unwrap().plan, @@ -843,7 +844,7 @@ mod tests { let mut exec_arena = crate::execution::ExecArena::new(context, &transaction); let mut execution_arena = crate::execution::ExecContext::new(&mut exec_buffer, &mut exec_arena); - let mut ranges = ScanRanges::Owned(Range::Dummy); + let mut ranges = Cow::Owned(Range::Dummy); IndexScan::bind_ranges( info, info.lookup.as_ref().expect("lookup"), @@ -852,7 +853,7 @@ mod tests { &mut ranges, )?; assert_eq!( - ranges.range(), + &*ranges, &Range::Scope { min: Bound::Included( DataValue::Tuple(vec![DataValue::Int32(2), DataValue::Int32(2),]).into() diff --git a/src/execution/dql/index_scan.rs b/src/execution/dql/index_scan.rs index 474a0cdb..95e6d85b 100644 --- a/src/execution/dql/index_scan.rs +++ b/src/execution/dql/index_scan.rs @@ -23,30 +23,18 @@ use crate::planner::operator::SortOption; use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::{IndexScanView, StorageTransaction, TupleMapping}; use crate::types::index::{IndexInfo, IndexLookup, RuntimeIndexProbe}; +use std::borrow::Cow; pub(crate) struct IndexScan<'a> { op: &'a TableScanOperator, info: &'a IndexInfo, lookup: &'a IndexLookup, iter: IterSlot, - ranges: ScanRanges<'a>, + /// Borrows the plan's static range, or owns one that parameters bind into. + ranges: Cow<'a, Range>, cover_mapping: Option, } -pub(crate) enum ScanRanges<'a> { - Static(&'a Range), - Owned(Range), -} - -impl ScanRanges<'_> { - pub(crate) fn range(&self) -> &Range { - match self { - ScanRanges::Static(range) => range, - ScanRanges::Owned(range) => range, - } - } -} - impl<'a> IndexScan<'a> { pub(crate) fn new( op: &'a TableScanOperator, @@ -70,7 +58,7 @@ impl<'a> IndexScan<'a> { info, lookup, iter, - ranges: ScanRanges::Owned(Range::Dummy), + ranges: Cow::Owned(Range::Dummy), cover_mapping, } } @@ -81,11 +69,11 @@ impl<'a> IndexScan<'a> { lookup: &'a IndexLookup, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut (dyn MetaArena + 'a), - ranges: &mut ScanRanges<'a>, + ranges: &mut Cow<'a, Range>, ) -> Result<(), DatabaseError> { match lookup { IndexLookup::Static(template) if !template.has_parameter() => { - *ranges = ScanRanges::Static(template) + *ranges = Cow::Borrowed(template) } IndexLookup::Static(template) => { let plan_arena = &*plan_arena; @@ -95,19 +83,19 @@ impl<'a> IndexScan<'a> { }) }; match ranges { - ScanRanges::Owned(range) => bind(range)?, - ScanRanges::Static(_) => { + Cow::Owned(range) => bind(range)?, + Cow::Borrowed(_) => { let mut range = Range::Dummy; bind(&mut range)?; - *ranges = ScanRanges::Owned(range); + *ranges = Cow::Owned(range); } } } IndexLookup::Probe => match (&mut *ranges, arena.pop_runtime_probe()) { - (ScanRanges::Owned(Range::Eq(value)), RuntimeIndexProbe::Eq(probe)) => { + (Cow::Owned(Range::Eq(value)), RuntimeIndexProbe::Eq(probe)) => { *value = RangeValue::Value(probe) } - (ranges, probe) => *ranges = ScanRanges::Owned(probe.into()), + (ranges, probe) => *ranges = Cow::Owned(probe.into()), }, } if let ( @@ -119,12 +107,12 @@ impl<'a> IndexScan<'a> { { if let Some(specialized) = RangeDetacher::::specialize_range( info.meta, - ranges.range(), + ranges, predicate, *ignore_prefix_len, plan_arena, )? { - *ranges = ScanRanges::Owned(specialized); + *ranges = Cow::Owned(specialized); } } Ok(()) @@ -190,7 +178,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexScan<'a> { } let view = IndexScanView { - range: self.ranges.range(), + range: &self.ranges, cover_mapping: self.cover_mapping.as_ref(), }; let state = arena.local_state(plan_arena); From 2fa92fcb4453141e8af008cdd1ad8a437327a8f9 Mon Sep 17 00:00:00 2001 From: kould Date: Fri, 6 Nov 2026 19:10:53 +0800 Subject: [PATCH 13/14] chore: document the safety contract of MappedScope::new Fixes clippy's missing_safety_doc and drops the doc comment of Tuple::deserialize_from_into. --- src/types/mapped.rs | 4 ++++ src/types/tuple.rs | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/types/mapped.rs b/src/types/mapped.rs index 011f7ac5..c701b6f7 100644 --- a/src/types/mapped.rs +++ b/src/types/mapped.rs @@ -32,6 +32,10 @@ pub struct MappedScope { } impl MappedScope { + /// # Safety + /// + /// The scope must belong to one transaction whose reads stay valid, unchanged and at the same + /// address until the scope is dropped, and it must be dropped before that transaction ends. pub unsafe fn new() -> Self { Self { #[cfg(debug_assertions)] diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 1a05c9bf..2e0a0bed 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -747,9 +747,6 @@ impl Tuple { Tuple { pk, values } } - /// Decodes a stored row. With a `scope` strings point into `bytes`, otherwise they are - /// copied. - /// /// # Safety /// /// `bytes` must be a row this crate encoded, since strings are not checked again as UTF-8. From 7ee772995939ca4d05d70274b1ad1adabe09ca16 Mon Sep 17 00:00:00 2001 From: kould Date: Fri, 6 Nov 2026 19:10:53 +0800 Subject: [PATCH 14/14] docs: add the zero-copy SQL on LMDB article in English and Chinese --- docs/zero-copy-sql-on-lmdb.md | 387 ++++++++++++++++++++++++++++ docs/zero-copy-sql-on-lmdb.zh-CN.md | 387 ++++++++++++++++++++++++++++ 2 files changed, 774 insertions(+) create mode 100644 docs/zero-copy-sql-on-lmdb.md create mode 100644 docs/zero-copy-sql-on-lmdb.zh-CN.md diff --git a/docs/zero-copy-sql-on-lmdb.md b/docs/zero-copy-sql-on-lmdb.md new file mode 100644 index 00000000..f116958e --- /dev/null +++ b/docs/zero-copy-sql-on-lmdb.md @@ -0,0 +1,387 @@ +# Zero-Copy SQL Computation on LMDB + +English | [中文](zero-copy-sql-on-lmdb.zh-CN.md) + +> By the time a database hands a row to your application, its strings have usually been copied several times: from the file into a cache, from the cache into the executor, and from the executor into your struct. This article explores another option. If the storage layer is an mmap-based KV engine such as LMDB, the pages a read-only transaction reads keep their addresses until the transaction ends. Can a SQL engine then let query results borrow those pages all the way through, so that the whole SQL computation runs without copying? + +## Summary + +SQLite is the most common in-process SQL database, with stable bindings in most languages. This example uses Rust's rusqlite. A very common task: fetch all comments under a thread, map them to structs, assemble them into a reply tree by `parent_id`, and serialize the tree as nested JSON for the caller: + +```rust +use rusqlite::{params, Connection}; +use serde::Serialize; +use std::collections::HashMap; +use std::error::Error; +use std::io::Write; + +#[derive(Serialize)] +struct Post { + id: i64, + thread_id: i64, + parent_id: i64, // 0 means a top-level comment + user: String, + body: String, +} + +#[derive(Serialize)] +struct Comment<'a> { + #[serde(flatten)] + post: &'a Post, + replies: Vec>, +} + +fn build<'a>(children: &'a HashMap>, parent: i64) -> Vec> { + let Some(posts) = children.get(&parent) else { return Vec::new() }; + posts.iter() + .map(|post| Comment { post, replies: build(children, post.id) }) + .collect() +} + +fn write_thread(conn: &Connection, thread_id: i64, out: impl Write) -> Result<(), Box> { + let mut stmt = conn.prepare_cached("select id, thread_id, parent_id, user, body from posts where thread_id = ?1")?; + let mut rows = stmt.query(params![thread_id])?; + // Building the tree needs every row, but a row only lives until the next call to next, so Post must own its Strings + let mut children: HashMap> = HashMap::new(); + while let Some(row) = rows.next()? { + let post = Post { + id: row.get(0)?, + thread_id: row.get(1)?, + parent_id: row.get(2)?, + user: row.get(3)?, + body: row.get(4)?, + }; + children.entry(post.parent_id).or_default().push(post); + } + serde_json::to_writer(out, &build(&children, 0))?; + Ok(()) +} +``` + +For this utterly ordinary "fetch rows, map them to structs" code, the data goes through these steps before it becomes the `Post` in your hands: + +```text + Database file + │ + │ Default: read() copies whole pages into SQLite's page cache [copy] + │ - With mmap_size: read the mapped file pages directly + ▼ + B-tree page (page cache or file mapping) + │ + │ sqlite3_step → OP_Column decodes user / body from the record + │ - Within a page: copy into a register buffer [copy] + │ - Across overflow pages: malloc a buffer, join page by page [alloc][copy] + │ With ORDER BY / GROUP BY: write into the sorter and read back [copy] + ▼ + Register buffer + │ + │ sqlite3_column_text → rusqlite row.get() + │ The pointer only lives until the next call to next, so Post must own a copy + │ user / body each become a String [alloc][copy] × 2 + ▼ + Post { user: String, body: String, .. } + │ + │ Build the tree by parent_id and write it out recursively + │ serde_json::to_writer writes the JSON + ▼ + JSON +``` + +Along this path, both SQLite and its API contract require strings to be moved somewhere else. The more fields, the longer the strings and the more complex the query, the more copies and allocations happen along the way. This is in fact the standard design of most databases: manage your own buffer pool, control memory in user space, and don't let the operating system decide which pages stay in memory. The cost is that a page in the pool can be evicted at any moment, so data must be copied before it is handed out. If, as DBOS [1] envisions, the database moved into the operating system kernel and managed the OS state itself, the user-space buffer pool and the kernel page cache might no longer need to be two separate layers. + +Yet in typical web applications, the main work of many queries is just shaping and passing data along: take data out of SQL, convert it into the corresponding structs (POJOs in Java) with some light processing, serialize it to JSON, and return it over HTTP. GraphQL [2] turned this into an interface specification: the caller describes the fields and nesting it wants, and the server takes the columns out of the database in that shape, assembles them and serializes them, doing almost nothing else. This kind of nearly pure read-and-reshape work makes up a growing share of web backends. + +Here, every `String` field of `Post` costs one allocation, and more fields mean more allocations. Look back at the flow above and you'll see each string is copied twice. The first copy happens when SQLite's `step` reads the column and copies it from the page into a register buffer; even with mmap enabled, this copy can't be avoided. The second happens when mapping to `Post`, copying from the register buffer into each field. But `row.get_ref` can lend out a `&str` directly. Why not let `Post`'s fields borrow it and skip the second copy? Unfortunately, `row` only lives until the next call to `next`, so a `Post` borrowing from it can't outlive the row, and the compiler rejects it outright. Building the tree needs all rows at once, and neither SQL sorting nor JSON functions can produce nesting of arbitrary depth, so these rows must all be held at the same time. This copy comes from SQLite's API contract, and code on top of it can hardly avoid it. + +## Zero-Copy Computation from Storage to Results + +### MMAP Storage + +Andy Pavlo once said that MMAP will ruin your database and your life [3]. That was one of the rejected title candidates for his CIDR 2022 paper with Crotty and Leis, finally titled *Are You Sure You Want to Use MMAP in Your Database Management System?*. The paper lists four classes of problems with using mmap instead of a buffer pool in a database: transactional safety, I/O stalls, error handling and performance. As the title suggests, this work does not call mmap directly to build its own storage. It uses LMDB as the storage backend instead. LMDB is a mature embedded KV storage engine whose design avoids the most serious problem in the paper, transactional safety [4]: + +- **Read-only mapping by default.** The file is mapped read-only, so the process cannot modify it through pointers. Stray pointer writes in the host program can't corrupt the database, and the mapping holds no dirty pages the kernel could write back on its own. Trying to modify a value returned by a read-only transaction triggers SIGSEGV immediately. +- **Writes bypass the mapping.** A write transaction modifies its pages in heap memory first, writes them back to the file with `pwrite` at commit, then calls `fsync` as configured. LMDB alone decides when and in what order data reaches disk. +- **Copy-on-write B+ tree (shadow paging).** Committed pages are never modified in place. Changing a leaf writes a new page and copies every page on the path up to the root. +- **Two alternating meta pages.** The last step of a commit writes a meta page that records the new root. The two copies are written alternately, so after a crash one of them is always complete, and the database never stops in a half-written state. +- **A reader table protects snapshots.** Every read-only transaction registers the version it sees, and old pages it might read are never reclaimed for reuse. So a value read by a read-only transaction stays valid at the same address until the transaction ends. This is exactly the guarantee that **zero-copy SQL computation** relies on. +- **A single writer.** Only one write transaction runs at a time, and the paper notes that this is how LMDB avoids conflicts between transactions. + +LMDB does not solve the paper's other problems, of course: a page fault blocks the thread synchronously, I/O errors turn into SIGBUS, and the kernel decides which pages stay in memory. LMDB assumes the hot data fits in memory, and in the embedded scenarios SQLite targets, that assumption usually holds. + +### Data Encode + +The first step in building SQL on a KV store is deciding how a row becomes a key and a value. TiDB's approach is representative [5]: + +```text +Row: t{TableID}_r{RowID} → [col1, col2, col3, ...] +Index: t{TableID}_i{IndexID}_{indexed column values}_{RowID} → null +``` + +Rows of the same table share a prefix and are sorted by primary key, so both full table scans and index lookups become range scans over contiguous keys. The implementation here uses the same structure: the key is "table ID + type tag + primary key", and the value holds the row's columns. + +For zero copy, what matters is how strings are stored in the value. Take the `orders` table used in the benchmarks below. After inserting the row `(1, 7, 'paid', 42, 'hello')`, its value looks like this: + +```text + 00 null bitmap + 01 00 00 00 id = 1 + 07 00 00 00 customer_id = 7 + 04 00 00 00 70 61 69 64 status: length 4 + "paid" + 2a 00 00 00 amount = 42 + 05 00 00 00 68 65 6c 6c 6f note: length 5 + "hello" + 00 00 00 00 00 00 00 00 statement stamp: the id of the statement that wrote this row, used by updates to skip rows the same statement just wrote +``` + +Strings are stored contiguously as "length + raw bytes". Decoding `note` only needs to read its 4-byte length to get a (pointer, length) pair pointing into an LMDB page, which is exactly a `&str` with no conversion at all. LMDB also guarantees that the overflow pages holding a large value are contiguous, so however long the string is, its bytes form one contiguous block in the mapping and can be lent out directly. Note that all columns of a row form a single LMDB value. LMDB places a value that doesn't fit in a leaf on a contiguous run of overflow pages, so the whole row is contiguous, not just one field: `status` and `note` are simply two offsets into the same bytes. + +Putting large values on separate pages follows the same idea as PostgreSQL's TOAST and InnoDB's external pages. Those mechanisms solve row bloat, so queries that read only small columns don't touch large values. They don't solve copying, though: values are still copied when fetched into the application. The truly copy-free approach is something like FILESTREAM, handing the value out as a file handle, but then it is no longer a SQL value. The engine can no longer filter, sort or measure it, and it no longer lines up with the row's transaction snapshot. The difference here is that overflow pages stay in the same mapping and the same read-only transaction, so operators and the caller see the same bytes. + +### Execution + +Lending strings out of storage is only the first step. A SQL statement passes through a chain of operators, such as scan, filter, sort, group and join, before its results reach the caller, and if any layer copies a string, zero copy breaks there. The approach is a discipline: storage provides a lifetime, operators don't break it, and Rust's borrow checker lets this be enforced at compile time. + +**Value representation: either owned or borrowed.** Strings and binary values use the same type (simplified): + +```rust +pub struct MaybeMapped(Repr); + +enum Repr { + Owned(B::Owned), // memory it owns: literals in SQL, new values computed by expressions + Mapped { ptr, len }, // a (pointer, length) pair pointing into a page read by a read-only transaction +} +``` + +Both states appear in the same query. In `where note = 'hello'`, for instance, the constant `'hello'` is `Owned` and the column value `note` is `Mapped`. + +When a column value is `Mapped` depends on the transaction. A read-only transaction declares at its start that "the bytes it reads stay valid at the same address until it ends", so row decoding produces `Mapped` directly. A write transaction can't promise this, so decoding has to copy and produces `Owned`. + +**Materializing operators copy only pointers.** A materializing operator must collect rows from upstream: sort gathers all rows, grouping keeps group keys, hash join builds a hash table, and window functions hold an entire partition. When they keep values, a `Mapped` string copies only its (pointer, length), not its content. The whole query runs in one read-only transaction and those pages never become invalid, so as long as nothing spills, materializing operators can keep values in their in-memory containers as borrows. New strings appear only in a few cases: values computed by expressions (such as `upper(note)`), and values serialized out and read back when the data is large enough to spill. In principle, spilled files could also be read back through mmap and borrowed again, but that is extra work. + +**How long borrows last: enforced by the compiler outside, by convention inside.** The risk of borrowing is a pointer outliving its page. This is handled in two layers: + +- Results handed to the caller are tied to the transaction's lifetime. The `Post` in the conclusion works this way: it can outlive any single row and be used until the transaction ends, but trying to carry it out of the transaction is rejected by the compiler. +- Borrows inside the executor rely on convention. The same prepared statement can run repeatedly in different transactions, so before each execution, operators must clear the borrowed values left from the previous run. The compiler can't check this layer; a liveness check in debug builds backs it up, panicking on any read of a borrowed value whose transaction has ended. + +## Benchmarking SQL Execution: SQLite with mmap vs. LMDB SQL + +In the benchmarks below, both sides load identical data, build the same indexes and run the same SQL. Each statement is prepared once, and all executions run in one read-only transaction. SQLite has `PRAGMA mmap_size` enabled (1 GiB, larger than the database file, so the whole file is mapped), so both sides read the database file through a memory mapping, and the I/O conditions are the same. + +- **LMDB SQL**: the SQL engine running on LMDB read-only transactions, with strings borrowed from storage all the way to the caller. +- **LMDB SQL (forced copy)**: the same query run in a write transaction, so strings are copied. This is the control group. +- **SQLite**: SQLite 3.45.1 with mmap enabled. + +Allocations are counted at libc's `malloc`. The machine is an i9-13900HX, with the program pinned to one P-core (`cpu8`). + +The goal is not to prove which side is faster, but to find the string length at which the cost of copying starts to show. So string lengths range from 64 B up to 256 KiB. At 64 B SQLite leads, and 256 KiB is far larger than a typical column in a business table; the tiers in between are the common document, log and JSON fields. + +Tables 1 to 3 use the `orders` table below. `note` is loaded at lengths from 64 B to 256 KiB, each tier in its own database, and each query hits about 10 rows by `(customer_id, status)`. + +```sql +create table orders ( + id int primary key, + customer_id int, + status varchar, -- 'new' / 'paid' / 'shipped' / 'done' / 'void' + amount int, + note varchar +); +create index orders_customer_status on orders (customer_id, status); +``` + +**Table 1: Plain index lookup** + +Fetch about 10 rows through the index, and the caller reads every string. Reproduce with `examples/alloc_trace.rs` [6]. + +```sql +select id, status, amount, note from orders +where customer_id = $1 and status = $2 and amount > $3; +``` + +| note length | LMDB SQL | LMDB SQL (forced copy) | SQLite | +| ---: | ---: | ---: | ---: | +| 64 B | 3.74 / 0.4 | 5.22 / 1.1 | 2.99 / 0 | +| 1 KiB | 3.70 / 0.4 | 5.33 / 10 | 3.44 / 0 | +| 4 KiB | 4.37 / 0.4 | 8.00 / 40 | 6.44 / 40 | +| 16 KiB | 3.65 / 0.4 | 13.07 / 160 | 11.60 / 158 | +| 64 KiB | 2.56 / 0.4 | 18.57 / 640 | 18.05 / 628 | +| 256 KiB | 2.38 / 0.4 | 61.58 / 2560 | 66.14 / 2458 | + +Unit: each cell is "p50 latency µs / KiB allocated per execution". + +**Table 2: Length functions** + +Same predicate and data as Table 1, with only the projection replaced by a length function. The first column fetches `note` and has the caller call `len()` on the string; `char_length` corresponds to `length()` in SQLite. + +| note length | `select note`, caller calls `len()` | `octet_length(note)` | `char_length(note)` | +| ---: | ---: | ---: | ---: | +| 64 B | 3.56 / 2.52 | 3.81 / 2.51 | 3.93 / 2.77 | +| 4 KiB | 4.33 / 6.87 | 4.77 / 2.93 | 7.34 / 16.07 | +| 64 KiB | 2.35 / 20.35 | 2.75 / 1.95 | 26.07 / 164.01 | +| 256 KiB | 2.14 / 69.03 | 2.35 / 1.82 | 89.25 / 627.00 | + +Unit: each cell is "LMDB SQL / SQLite p50 latency µs". + +**Table 3: Allocation and performance across several materializing operators** + +Read the same rows as Table 1, but pass the strings through three materializing operators in turn: grouping, a window function and a sort. Reproduce with the `multi-op` variant in `examples/alloc_trace.rs` [6]. + +```sql +select status, note, peak, max(peak) over (partition by status) +from (select status, note, max(amount) as peak from orders + where customer_id = $1 and status = $2 and amount > $3 + group by status, note) t +order by note desc; +``` + +| note length | LMDB SQL | SQLite | +| ---: | ---: | ---: | +| 64 B | 7.91 / 4.2 | 12.74 / 109 | +| 4 KiB | 13.82 / 4.2 | 33.19 / 757 | +| 64 KiB | 84.74 / 4.1 | 328.34 / 10846 | +| 256 KiB | 320.24 / 4.0 | 7479.64 / 32945 | + +Unit: each cell is "p50 latency µs / KiB allocated per execution". + +**Table 4: Parsing 100 JSON / CSV documents** + +Each execution fetches 100 documents by `grp`, keeps the results past the end of the query, then parses them into structs. On LMDB SQL the struct fields borrow the text in the transaction directly; on SQLite they follow the common pattern of one `String` per field. The CSV case uses a `csv_docs` table with the same structure, where `body` holds one CSV line. Reproduce with `examples/borrowed_parse.rs` [7]. + +```sql +create table json_docs ( + id int primary key, + grp int, -- 100 documents per group + body varchar +); +create index json_docs_grp on json_docs (grp); + +select id, body from json_docs where grp = $1; +``` + +| Document size | Case | LMDB SQL | SQLite | +| ---: | --- | ---: | ---: | +| 256 B | Keep raw rows only | 28.34 / 3.5 | 24.64 / 28 | +| 256 B | Parse JSON | 43.27 / 12 | 60.17 / 35 | +| 2 KiB | Parse JSON | 68.80 / 12 | 97.20 / 214 | +| 16 KiB | Keep raw rows only | 28.60 / 3.5 | 137.69 / 3205 | +| 16 KiB | Parse JSON | 187.33 / 12 | 292.65 / 3212 | +| 16 KiB | Parse CSV | 35.15 / 12 | 172.15 / 3206 | + +Unit: each cell is "p50 latency µs / KiB allocated per execution", with 100 rows per execution. + +### Analysis + +**LMDB SQL's allocations don't depend on data size; SQLite's follow the data it returns.** In Table 1, LMDB SQL always allocates 0.4 KiB. From 4 KiB on, SQLite allocates once per row, and the total roughly equals the bytes returned. SQLite shows 0 at 64 B and 1 KiB, but that doesn't mean nothing was copied: short strings are copied into a reused buffer, so no allocation is counted. Checking the returned pointers shows that every LMDB SQL pointer falls inside the mapping of the database file, and none of SQLite's do. + +**SQLite is fast on short data, but its latency grows with length; LMDB SQL's latency doesn't depend on length.** At 64 B SQLite needs only 2.99 µs, faster than LMDB SQL, because copying a few dozen bytes costs next to nothing. As `note` grows, SQLite's latency and allocations grow together, reaching 66.14 µs at 256 KiB, where copying becomes the main cost. LMDB SQL stays between 2.4 and 4.4 µs throughout. It doesn't read the string content, so length doesn't affect it. + +**The gap comes from copying, not I/O.** When LMDB SQL is forced to copy strings, its latency and allocations land in the same range as SQLite's (61.6 µs vs. 66.1 µs at 256 KiB). + +**`octet_length` is constant time on both sides, so it is no argument for zero copy.** LMDB SQL calls `len()` on the borrowed `&str`. For a column used only as the argument of a length function, SQLite reads the length straight from the record header without loading the content or reading overflow pages (`OPFLAG_BYTELENARG` on `OP_Column`), so SQLite is actually faster here. + +**`char_length` scans every byte on both sides, and from 64 KiB up LMDB SQL is 6 to 7 times faster.** LMDB SQL counts directly on the mapped pages (`chars().count()`, about 28 GB/s at 256 KiB). SQLite first joins the overflow page content into a `malloc`ed buffer, then counts characters one by one. This is the typical "read all the content, process it lightly" case: the computation is cheap, copying dominates, and zero copy pays off the most. Recording the character count or an "ASCII only" flag at write time would make `char_length` constant time too; this is not implemented yet. + +**Every materializing operator widens the gap again.** At 64 KiB and 256 KiB in Table 3, SQLite allocates 13 to 17 times the returned data, because every operator writes records in and reads them back out. LMDB SQL allocates only about 4 KiB for its hash table and sort array. LMDB SQL's latency still grows with `note`, because grouping hashes the whole `note`. + +**Parsing itself is equally fast on both sides.** For 16 KiB JSON, after subtracting the fetch time, pure parsing takes about the same on both sides (159 µs vs. 155 µs). Nearly all of LMDB SQL's advantage comes from not copying during the fetch. + +## Conclusion + +Back to the code example from the beginning. The same functionality can be written like this: + +```rust +use kite_sql::db::{Transaction, TxnResultIter}; +use kite_sql::storage::lmdb::LmdbStorage; +use kite_sql::Model; +use serde::Serialize; +use std::borrow::Cow; +use std::collections::HashMap; +use std::error::Error; +use std::io::Write; + +#[derive(Default, Model, Serialize)] +#[model(table = "posts")] +struct Post<'a> { + #[model(primary_key)] + id: i32, + thread_id: i32, + parent_id: i32, // 0 means a top-level comment + user: Cow<'a, str>, + body: Cow<'a, str>, +} + +#[derive(Serialize)] +struct Comment<'a> { + #[serde(flatten)] + post: &'a Post<'a>, + replies: Vec>, +} + +fn build<'a>(children: &'a HashMap>>, parent: i32) -> Vec> { + let Some(posts) = children.get(&parent) else { return Vec::new() }; + posts.iter() + .map(|post| Comment { post, replies: build(children, post.id) }) + .collect() +} + +fn write_thread(tx: &mut Transaction<'_, LmdbStorage>, thread_id: i32, out: impl Write) -> Result<(), Box> { + // Post borrows LMDB pages and lives until the transaction ends, so all rows can be collected before building the tree + let mut children: HashMap> = HashMap::new(); + let posts = tx + .bind(|ctx| { + ctx.from::()? + .filter(|e| e.column(Post::thread_id())?.eq(thread_id))? + .finish() + })? + // Every string field is Cow::Borrowed and points straight into an LMDB page + .txn_orm::(); + for post in posts { + let post = post?; + children.entry(post.parent_id).or_default().push(post); + } + serde_json::to_writer(out, &build(&children, 0))?; + Ok(()) +} +``` + +Here the same process looks like this: + +```text + Database file (LMDB mapping) + │ + │ Read-only transaction: pages stay valid at the same address until it ends + ▼ + B+ tree page / overflow pages (a large value is one contiguous block in the mapping) + │ + │ Row decoding: read the 4-byte length, get (pointer, length) + ▼ + Cow::Borrowed(&str), pointing straight into the page + │ + │ txn_orm: put into the fields of Post + ▼ + Post { user: Cow::Borrowed, body: Cow::Borrowed, .. } (lives until the transaction ends) + │ + │ Build the tree by parent_id: only pointers move + │ serde_json::to_writer writes the JSON + ▼ + JSON +``` + +Apart from the types, the two programs are nearly identical: read `Post`s row by row, put them into a `HashMap` by `parent_id`, then build the tree recursively and serialize it. The difference is that this map holds only pointers. `Post`'s string fields are `Cow<'_, str>`, and in a read-only transaction they are all `Cow::Borrowed`, pointing straight into LMDB pages. Their lifetime is tied to the transaction rather than to a row, so the first row's `Post` can wait in the map alongside the last one until the tree is built and the JSON is written, and no string is copied even once. The fields are `Cow` rather than `&str` because pages may change in a write transaction, in which case the fields fall back to `Cow::Owned` copies. It's not that rusqlite can't borrow: `row.get_ref` lends out a `&str`, but only until the next call to `next`. Building the tree needs every row, and a `Post` borrowing from `row` can't survive the next `next`, so only copies can go into the map. + +This advantage has its limits. On workloads with short rows processed one at a time, the two sides are about even, with SQLite slightly faster. Strings are still copied in write transactions. Workloads like TPC-C show no change. When an aggregation operator holds borrowed rows, the pages behind them must stay in memory, but they are clean file pages that SQLite would also load into its page cache to read the same data, and the kernel can reclaim them at any time. Heap memory that can't be reclaimed is actually lower on the zero-copy side (Table 3). A read-only transaction has to stay open as long as the borrowed results live, and LMDB can't reclaim old pages while readers reference them, so holding results for a long time under heavy writes makes the database file grow. + +Thanks for reading. The implementation and benchmark code are in the KiteSQL repository: . + +## References + +[1] Athinagoras Skiadopoulos, Qian Li, Peter Kraft, Kostis Kaffes, Daniel Hong, Shana Mathew, David Bestor, Michael Cafarella, Vijay Gadepally, Goetz Graefe, Jeremy Kepner, Christos Kozyrakis, Tim Kraska, Michael Stonebraker, Lalith Suresh, Matei Zaharia. *DBOS: A DBMS-oriented Operating System*. PVLDB, 15(1): 21–30, 2022. doi:[10.14778/3485450.3485454](https://doi.org/10.14778/3485450.3485454). PDF: + +[2] GraphQL Specification, September 2025 Edition. GraphQL Foundation. . Project site: + +[3] Andrew Crotty, Viktor Leis, Andrew Pavlo. *Are You Sure You Want to Use MMAP in Your Database Management System?* CIDR 2022. Project page: . PDF: . The candidate title is mentioned at 00:52 in Andy Pavlo's lecture recording: . + +[4] LMDB API documentation, Symas Corporation: . The `MDB_val` section says returned values are "valid only until a subsequent update operation, or the end of the transaction" and "commonly point into the database itself"; the `MDB_WRITEMAP` section explains that the default read-only mapping protects the database from stray pointer writes. + +[5] PingCAP. *TiDB Computing*, section "Mapping table data to Key-Value": + +[6] `examples/alloc_trace.rs`: . Run: `cargo run --release --example alloc_trace --features lmdb,orm` + +[7] `examples/borrowed_parse.rs`: . Run: `cargo run --release --example borrowed_parse --features lmdb,orm` diff --git a/docs/zero-copy-sql-on-lmdb.zh-CN.md b/docs/zero-copy-sql-on-lmdb.zh-CN.md new file mode 100644 index 00000000..30f22d14 --- /dev/null +++ b/docs/zero-copy-sql-on-lmdb.zh-CN.md @@ -0,0 +1,387 @@ +# 基于 LMDB 实现零拷贝的 SQL 计算 + +[English](zero-copy-sql-on-lmdb.md) | 中文 + +> 数据库把一行数据交到应用程序手里之前，字符串通常已经被拷贝过好几次：从文件到缓存，从缓存到执行器，再从执行器到你的结构体。这篇文章讨论的是另一种可能：如果存储层是 LMDB 这样基于 mmap 的 KV 引擎，只读事务读到的页在事务结束前地址不变，那么 SQL 引擎能不能让查询结果一路借用这些页，让完整的 SQL 计算做到零拷贝？ + +## Summary + +SQLite 是应用进程内最常见的 SQL 数据库，在常见语言中都有稳定的绑定 API，这里以 Rust 的 rusqlite 为例。一个很常见的场景是：取出一个帖子下的全部评论，映射成结构体，按 `parent_id` 组装成回复树，再序列化成嵌套的 JSON 写给调用方： + +```rust +use rusqlite::{params, Connection}; +use serde::Serialize; +use std::collections::HashMap; +use std::error::Error; +use std::io::Write; + +#[derive(Serialize)] +struct Post { + id: i64, + thread_id: i64, + parent_id: i64, // 0 表示顶层评论 + user: String, + body: String, +} + +#[derive(Serialize)] +struct Comment<'a> { + #[serde(flatten)] + post: &'a Post, + replies: Vec>, +} + +fn build<'a>(children: &'a HashMap>, parent: i64) -> Vec> { + let Some(posts) = children.get(&parent) else { return Vec::new() }; + posts.iter() + .map(|post| Comment { post, replies: build(children, post.id) }) + .collect() +} + +fn write_thread(conn: &Connection, thread_id: i64, out: impl Write) -> Result<(), Box> { + let mut stmt = conn.prepare_cached("select id, thread_id, parent_id, user, body from posts where thread_id = ?1")?; + let mut rows = stmt.query(params![thread_id])?; + // 建树要等所有行到齐，而 row 只活到下一次 next，所以 Post 只能拥有 String + let mut children: HashMap> = HashMap::new(); + while let Some(row) = rows.next()? { + let post = Post { + id: row.get(0)?, + thread_id: row.get(1)?, + parent_id: row.get(2)?, + user: row.get(3)?, + body: row.get(4)?, + }; + children.entry(post.parent_id).or_default().push(post); + } + serde_json::to_writer(out, &build(&children, 0))?; + Ok(()) +} +``` + +这样一段再普通不过的"取行、映射成结构体"的代码，数据要经过下面这些步骤，才能变成你手里的 `Post`： + +```text + 数据库文件 + │ + │ 默认：read() 把整页拷进 SQLite 的 page cache [拷贝] + │ - 开启 mmap_size：直接读映射进来的文件页 + ▼ + B 树页（page cache 或文件映射） + │ + │ sqlite3_step → OP_Column 从记录中解出 user / body + │ - 在页内：拷进寄存器缓冲区 [拷贝] + │ - 跨溢出页：malloc 缓冲区，逐页拼接 [分配][拷贝] + │ 有 ORDER BY / GROUP BY 时：写进 sorter 再读出 [拷贝] + ▼ + 寄存器缓冲区 + │ + │ sqlite3_column_text → rusqlite row.get() + │ 指针只活到下一次 next，Post 只能拥有一份拷贝 + │ user / body 各自变成 String [分配][拷贝] × 2 + ▼ + Post { user: String, body: String, .. } + │ + │ 按 parent_id 建树，递归写出 + │ serde_json::to_writer 写出 JSON + ▼ + JSON +``` + +在这样的过程里，SQLite 和它的 API 约定都要求字符串换一个地方存放。字段越多、字符串越长、查询越复杂，这条路上的拷贝和分配就越多。这其实也是常见数据库的标准做法：自己管理 buffer pool，在用户态掌控内存，不让操作系统决定哪些页留在内存里。代价是 pool 里的页随时可能被替换，数据要交出去，就必须先拷贝一份。如果像 DBOS [1] 设想的那样，把数据库放进操作系统的内核，由它来管理操作系统的状态，用户态的 buffer pool 和内核的页缓存或许就不必再分成两层。 + +可是在常见的 Web 应用里，很多查询的主要工作其实只是数据的加工和传递：从 SQL 里取出数据，经过一些简单的处理转换成对应的结构体（Java 里的 POJO），再序列化成 JSON，通过 HTTP 返回。GraphQL [2] 把这件事做成了接口规范：调用方描述想要的字段和嵌套结构，服务端照着这个形状把数据库里的列取出来、组装好、序列化出去，几乎不做别的。这类几乎纯读取和重排的工作，在 Web 后端里占的比例越来越大。 + +在这样的情况下，`Post` 的每个 `String` 字段都要分配一次，字段越多，分配越多。再回头看刚才的流程图，你会发现字符串一共被拷贝了两次：第一次是 SQLite 在 `step` 取列时，把它从页里拷进寄存器缓冲区（即使启用了 mmap，这一次拷贝也省不掉）；第二次是映射成 `Post` 时，又从寄存器缓冲区拷进各个字段。但 `row.get_ref` 明明能直接借出 `&str`，让 `Post` 的字段借用它，不就省掉第二次拷贝了吗？遗憾的是，`row` 只活到下一次 `next`，借用它的 `Post` 带不出这一行，编译器会直接拒绝。而建树要等所有行到齐，SQL 的排序和 JSON 函数都给不出深度不定的嵌套结构，这些行必须同时拿在手里。这次拷贝来自 SQLite API 的约定，靠上层的写法很难绕开。 + +## 实现从存储到返回的零拷贝计算 + +### MMAP Storage + +Andy Pavlo 说过，MMAP 会毁掉你的数据库和你的生活 [3]。这是他和 Crotty、Leis 那篇 CIDR 2022 论文被否掉的候选标题之一，最后定名为 *Are You Sure You Want to Use MMAP in Your Database Management System?*。论文列举了在数据库里用 mmap 替代 buffer pool 的四类问题：事务安全、IO 停顿、错误处理、性能。正如标题所说，这里并没有直接调用 mmap 自己实现存储，而是用 LMDB 作为存储后端。LMDB 是一个成熟的嵌入式 KV 存储引擎，它的设计规避了论文里最致命的事务安全问题 [4]： + +- **默认只读映射。** 文件以只读方式映射，进程不能通过指针修改它。宿主程序的野指针写不坏数据库，映射里也不存在会被内核擅自写回磁盘的脏页；试图修改只读事务返回的值，会直接触发 SIGSEGV。 +- **写入不经过映射。** 写事务要修改的页先在堆内存里改好，提交时用 `pwrite` 写回文件，再按配置 `fsync`。什么时候落盘、按什么顺序落盘，由 LMDB 自己决定。 +- **写时复制的 B+ 树（影子分页）。** 已提交的页从不原地修改：改一个叶子，就写一份新页，并沿路径一直复制到根。 +- **两个元数据页交替使用。** 提交的最后一步是写元数据页，记下新的根。两份交替写，崩溃后总有一份是完整的，数据库不会停在写了一半的状态。 +- **读者表保护快照。** 每个只读事务都登记自己看到的版本，它可能读到的旧页不会被回收复用。所以只读事务读到的值，在事务结束前一直有效、地址不变，这正是 **零拷贝 SQL 计算** 所依赖的保证。 +- **单写者。** 同一时刻只有一个写事务，论文也提到 LMDB 正是靠这一点避免了事务之间的冲突。 + +当然，论文里的另一些问题 LMDB 并没有解决：缺页时线程会同步阻塞，IO 错误会变成 SIGBUS，哪些页留在内存由内核决定。LMDB 的前提是热数据放得进内存，而在 SQLite 所面向的嵌入式场景里，这个前提通常是成立的。 + +### Data Encode + +在 KV 存储之上实现 SQL，第一步是决定一行数据怎么变成 key 和 value。TiDB 的做法很有代表性 [5]： + +```text +行： t{TableID}_r{RowID} → [col1, col2, col3, ...] +索引：t{TableID}_i{IndexID}_{索引列的值}_{RowID} → null +``` + +同一张表的行共享同一个前缀，按主键排好序，全表扫描和索引查询都变成了一段连续 key 的范围扫描。这里的实现也是同样的结构：key 是"表 ID + 类型标记 + 主键"，value 是这一行的各列。 + +对零拷贝来说，关键在 value 里字符串怎么存。以后面测试里用到的 `orders` 表为例，插入 `(1, 7, 'paid', 42, 'hello')` 这一行，它的 value 是这样的： + +```text + 00 空值位图 + 01 00 00 00 id = 1 + 07 00 00 00 customer_id = 7 + 04 00 00 00 70 61 69 64 status：长度 4 + "paid" + 2a 00 00 00 amount = 42 + 05 00 00 00 68 65 6c 6c 6f note：长度 5 + "hello" + 00 00 00 00 00 00 00 00 语句戳：写入这一行的语句编号，更新时用来跳过本语句刚写入的行 +``` + +字符串以"长度 + 原始字节"的形式连续存放。解码 `note` 时只要读出 4 字节的长度，就得到一段指向 LMDB 页的（指针，长度），正好就是一个 `&str`，不需要任何转换。LMDB 又保证大值所在的溢出页是连续的，所以无论字符串多长，这段字节在映射里都是连续的一整块，可以直接借出去。注意一行的所有列是一个 LMDB value：LMDB 把放不进叶子的 value 分配在一段连续的溢出页上，所以不只是某个字段连续，而是整行连续，`status` 和 `note` 只是同一段字节里的两个偏移。 + +把大值放到单独的页上，和 PostgreSQL 的 TOAST、InnoDB 的外部页是同一个思路，解决的是行膨胀，让只读小列的查询不用碰大值。但这些机制不解决拷贝：值取回应用时照样要拷。真正不拷的做法是像 FILESTREAM 那样把值变成文件句柄交出去，可那样它就不再是 SQL 值了，引擎里不能再对它过滤、排序、求长度，和行的事务快照也对不齐。这里的区别在于，溢出页仍然在同一个映射、同一个只读事务里，算子和调用方拿到的是同一段字节。 + +### Execution + +字符串能从存储里借出来，只是第一步。一条 SQL 要经过扫描、过滤、排序、分组、join 等一连串算子，才会把结果交给调用方，其中任何一层把字符串拷贝一份，零拷贝就在那里断掉。这是一种风格：存储给出一个生命周期，算子不打断它，Rust 的借用检查让这件事能在编译期得到保证。 + +**值的表示：要么拥有，要么借用。** 字符串和二进制值都用同一个类型表示（简化后）： + +```rust +pub struct MaybeMapped(Repr); + +enum Repr { + Owned(B::Owned), // 自己拥有的内存：SQL 里的字面量、表达式算出来的新值 + Mapped { ptr, len }, // 一对（指针，长度），指向只读事务读到的页 +} +``` + +两种状态会出现在同一条查询里。比如 `where note = 'hello'`，常量 `'hello'` 是 `Owned`，列值 `note` 是 `Mapped`。 + +至于列值什么时候是 `Mapped`，取决于事务。只读事务开始时声明"读到的字节在事务结束前一直有效、地址不变"，行解码就直接产出 `Mapped`；写事务无法保证这一点，解码时只能拷贝，产出 `Owned`。 + +**物化算子只拷贝指针。** 物化算子必然要收集上游算子的行：排序要攒齐所有行，分组要保存分组键，hash join 要建哈希表，窗口函数要留住整个分区。它们保存值时，`Mapped` 字符串只拷贝（指针，长度），不拷贝内容。整条查询都在同一个只读事务里，这些页不会失效，所以只要没有触发 spill，物化算子就能在内存容器里以借用的方式保存值。只有少数情况会产生新的字符串：表达式算出来的值（比如 `upper(note)`），以及数据量大到需要 spill 时，序列化写出再读回来的值。理论上 spill 出去的文件也可以用 mmap 读回并继续借用，但这是一项额外的做法。 + +**借用的期限：对外靠编译器，对内靠约定。** 借用的风险在于指针活得比页久，这里分两层处理： + +- 交给调用方的结果，与事务的生命周期绑定。结论里的 `Post` 就是这样：它可以活过某一行，一直用到事务结束，但想把它带出事务，编译器会直接拒绝。 +- 执行器内部的借用靠约定。同一条预编译的语句可以在不同事务里反复执行，所以每次执行开始前，算子都要清空上一次留下的借用值。这一层编译器检查不到，由 debug 构建下的存活检查兜底：读到已结束事务的借用值会直接 panic。 + +## Benchmarking SQL Execution: SQLite with mmap vs. LMDB SQL + +下面的测试里，两边装载完全相同的数据、建同样的索引、执行同样的 SQL：语句只准备一次，所有执行都在同一个只读事务里完成。SQLite 开启了 `PRAGMA mmap_size`（1 GiB，大于数据库文件，整个文件都被映射），所以两边都通过内存映射读数据库文件，IO 这一层的条件是相同的。 + +- **LMDB SQL**：跑在 LMDB 只读事务上的 SQL 引擎，字符串从存储一路借用到调用方。 +- **LMDB SQL（强制拷贝）**：同一条查询放进写事务执行，字符串会被拷贝，用作对照组。 +- **SQLite**：SQLite 3.45.1，开启 mmap。 + +分配量在 libc 的 `malloc` 处统计。测试机为 i9-13900HX，程序绑定在同一个 P 核（`cpu8`）上运行。 + +测试的目的不是证明哪边更快，而是找出拷贝的成本从什么长度开始显现。所以字符串长度从 64 B 一直拉到 256 KiB：64 B 是 SQLite 领先的一档，256 KiB 则远超一般业务表里一列的大小，中间几档才是常见的文档、日志和 JSON 字段。 + +表 1 到表 3 使用下面的 `orders` 表。`note` 的长度从 64 B 到 256 KiB 分档装载，每档单独建库；每次查询按 `(customer_id, status)` 命中约 10 行。 + +```sql +create table orders ( + id int primary key, + customer_id int, + status varchar, -- 'new' / 'paid' / 'shipped' / 'done' / 'void' + amount int, + note varchar +); +create index orders_customer_status on orders (customer_id, status); +``` + +**表 1：普通索引查询** + +按索引取出约 10 行，调用方读取每个字符串。复现：`examples/alloc_trace.rs` [6]。 + +```sql +select id, status, amount, note from orders +where customer_id = $1 and status = $2 and amount > $3; +``` + +| note 长度 | LMDB SQL | LMDB SQL（强制拷贝） | SQLite | +| ---: | ---: | ---: | ---: | +| 64 B | 3.74 / 0.4 | 5.22 / 1.1 | 2.99 / 0 | +| 1 KiB | 3.70 / 0.4 | 5.33 / 10 | 3.44 / 0 | +| 4 KiB | 4.37 / 0.4 | 8.00 / 40 | 6.44 / 40 | +| 16 KiB | 3.65 / 0.4 | 13.07 / 160 | 11.60 / 158 | +| 64 KiB | 2.56 / 0.4 | 18.57 / 640 | 18.05 / 628 | +| 256 KiB | 2.38 / 0.4 | 61.58 / 2560 | 66.14 / 2458 | + +单位：每格为"p50 耗时 µs / 每次执行分配 KiB"。 + +**表 2：长度函数** + +查询条件和数据与表 1 相同，只把投影换成长度函数。第一列是取出 `note` 后由调用方对字符串取 `len()`；`char_length` 在 SQLite 里对应 `length()`。 + +| note 长度 | `select note`，调用方取 `len()` | `octet_length(note)` | `char_length(note)` | +| ---: | ---: | ---: | ---: | +| 64 B | 3.56 / 2.52 | 3.81 / 2.51 | 3.93 / 2.77 | +| 4 KiB | 4.33 / 6.87 | 4.77 / 2.93 | 7.34 / 16.07 | +| 64 KiB | 2.35 / 20.35 | 2.75 / 1.95 | 26.07 / 164.01 | +| 256 KiB | 2.14 / 69.03 | 2.35 / 1.82 | 89.25 / 627.00 | + +单位：每格为"LMDB SQL / SQLite 的 p50 耗时 µs"。 + +**表 3：多重物化算子下的数据分配与计算性能** + +读取与表 1 相同的行，但让字符串先后经过分组、窗口函数和排序三个物化算子。复现：`examples/alloc_trace.rs` 中的 `multi-op` 变体 [6]。 + +```sql +select status, note, peak, max(peak) over (partition by status) +from (select status, note, max(amount) as peak from orders + where customer_id = $1 and status = $2 and amount > $3 + group by status, note) t +order by note desc; +``` + +| note 长度 | LMDB SQL | SQLite | +| ---: | ---: | ---: | +| 64 B | 7.91 / 4.2 | 12.74 / 109 | +| 4 KiB | 13.82 / 4.2 | 33.19 / 757 | +| 64 KiB | 84.74 / 4.1 | 328.34 / 10846 | +| 256 KiB | 320.24 / 4.0 | 7479.64 / 32945 | + +单位：每格为"p50 耗时 µs / 每次执行分配 KiB"。 + +**表 4：100 篇 JSON / CSV 文档的解析** + +每次按 `grp` 取出 100 篇文档，把结果保留到查询结束之后，再解析成结构体。LMDB SQL 的结构体字段直接借用事务里的文本；SQLite 按常见写法，每个字段一个 `String`。CSV 场景用结构相同的 `csv_docs` 表，`body` 存一行 CSV。复现：`examples/borrowed_parse.rs` [7]。 + +```sql +create table json_docs ( + id int primary key, + grp int, -- 每组 100 篇 + body varchar +); +create index json_docs_grp on json_docs (grp); + +select id, body from json_docs where grp = $1; +``` + +| 文档大小 | 场景 | LMDB SQL | SQLite | +| ---: | --- | ---: | ---: | +| 256 B | 只保留原始行 | 28.34 / 3.5 | 24.64 / 28 | +| 256 B | 解析 JSON | 43.27 / 12 | 60.17 / 35 | +| 2 KiB | 解析 JSON | 68.80 / 12 | 97.20 / 214 | +| 16 KiB | 只保留原始行 | 28.60 / 3.5 | 137.69 / 3205 | +| 16 KiB | 解析 JSON | 187.33 / 12 | 292.65 / 3212 | +| 16 KiB | 解析 CSV | 35.15 / 12 | 172.15 / 3206 | + +单位：每格为"p50 耗时 µs / 每次执行分配 KiB"，每次执行取 100 行。 + +### 表格分析 + +**LMDB SQL 的分配量和数据大小无关，SQLite 的分配量跟着返回的数据走。** 表 1 里 LMDB SQL 始终是 0.4 KiB；SQLite 从 4 KiB 起每行分配一次，总量约等于返回的字节数。SQLite 在 64 B 和 1 KiB 时显示 0，不表示没有拷贝：短字符串被拷进了一块复用的缓冲区，所以统计不到分配。检查返回的指针可以看到，LMDB SQL 的指针全部落在数据库文件的映射里，SQLite 的一个都没有。 + +**SQLite 在短数据上很快，但耗时随数据长度增长；LMDB SQL 的耗时和长度无关。** 64 B 时 SQLite 只要 2.99 µs，比 LMDB SQL 还快，拷贝几十个字节的代价可以忽略。随着 `note` 变长，SQLite 的耗时和分配量一起增长，到 256 KiB 时是 66.14 µs，拷贝成了主要开销。LMDB SQL 的耗时始终在 2.4 到 4.4 µs 之间，它不读字符串的内容，自然也不受长度影响。 + +**差距来自拷贝，不来自 IO。** 让 LMDB SQL 强制拷贝字符串后，它的耗时和分配都和 SQLite 落在同一个量级（256 KiB 时 61.6 µs 对 66.1 µs）。 + +**`octet_length` 两边都是常数时间，不能作为零拷贝的论据。** LMDB SQL 对借用的 `&str` 取 `len()`；SQLite 对只作为长度函数参数的列，直接从记录头读长度，不加载内容、不读溢出页（`OP_Column` 的 `OPFLAG_BYTELENARG`），所以反而更快。 + +**`char_length` 两边都要扫完全部字节，64 KiB 以上 LMDB SQL 快 6 到 7 倍。** LMDB SQL 直接在映射的页上计数（`chars().count()`，256 KiB 档约 28 GB/s）；SQLite 先把溢出页的内容拼进 `malloc` 的缓冲区，再逐字符计数。这是"读完全部内容、处理很轻"的典型情形：计算本身便宜，拷贝占了大头，零拷贝的收益最明显。如果写入时记下字符数或"是否纯 ASCII"，`char_length` 也能做到常数时间，目前没有实现。 + +**每经过一个物化算子，差距就放大一次。** 表 3 里 64 KiB 和 256 KiB 两档，SQLite 的分配量是返回数据的 13 到 17 倍，因为每个算子都要把记录写进去再读出来；LMDB SQL 只分配哈希表和排序数组用的约 4 KiB。LMDB SQL 的耗时仍然随 `note` 增长，原因是分组要对整条 `note` 求哈希。 + +**解析本身两边一样快。** 16 KiB 解析 JSON 时，扣掉取数据的耗时，纯解析两边差不多（159 µs 对 155 µs）。LMDB SQL 快出的部分，几乎全来自取数据时没有拷贝。 + +## 结论 + +回到一开始的代码案例中，在同样功能的代码下可以写成这样： + +```rust +use kite_sql::db::{Transaction, TxnResultIter}; +use kite_sql::storage::lmdb::LmdbStorage; +use kite_sql::Model; +use serde::Serialize; +use std::borrow::Cow; +use std::collections::HashMap; +use std::error::Error; +use std::io::Write; + +#[derive(Default, Model, Serialize)] +#[model(table = "posts")] +struct Post<'a> { + #[model(primary_key)] + id: i32, + thread_id: i32, + parent_id: i32, // 0 表示顶层评论 + user: Cow<'a, str>, + body: Cow<'a, str>, +} + +#[derive(Serialize)] +struct Comment<'a> { + #[serde(flatten)] + post: &'a Post<'a>, + replies: Vec>, +} + +fn build<'a>(children: &'a HashMap>>, parent: i32) -> Vec> { + let Some(posts) = children.get(&parent) else { return Vec::new() }; + posts.iter() + .map(|post| Comment { post, replies: build(children, post.id) }) + .collect() +} + +fn write_thread(tx: &mut Transaction<'_, LmdbStorage>, thread_id: i32, out: impl Write) -> Result<(), Box> { + // Post 借用 LMDB 的页，活到事务结束，所以可以先全部收齐再建树 + let mut children: HashMap> = HashMap::new(); + let posts = tx + .bind(|ctx| { + ctx.from::()? + .filter(|e| e.column(Post::thread_id())?.eq(thread_id))? + .finish() + })? + // 每个字符串字段都是 Cow::Borrowed，直接指向 LMDB 的页 + .txn_orm::(); + for post in posts { + let post = post?; + children.entry(post.parent_id).or_default().push(post); + } + serde_json::to_writer(out, &build(&children, 0))?; + Ok(()) +} +``` + +同样的过程，换到这里是这样的： + +```text + 数据库文件（LMDB 映射） + │ + │ 只读事务：页在事务结束前一直有效、地址不变 + ▼ + B+ 树页 / 溢出页（大值在映射里是连续的一整块） + │ + │ 行解码：读出 4 字节长度，得到（指针，长度） + ▼ + Cow::Borrowed(&str)，直接指向页 + │ + │ txn_orm：放进 Post 的字段 + ▼ + Post { user: Cow::Borrowed, body: Cow::Borrowed, .. }（活到事务结束） + │ + │ 按 parent_id 建树：只移动指针 + │ serde_json::to_writer 写出 JSON + ▼ + JSON +``` + +两段代码除了类型，几乎一模一样：逐行读出 `Post`，按 `parent_id` 放进 `HashMap`，最后递归建树并序列化。不同的是，这里的 map 里只有指针：`Post` 的字符串字段是 `Cow<'_, str>`，在只读事务里都是 `Cow::Borrowed`，直接指向 LMDB 的页。它们的生命周期绑在事务上而不是某一行上，所以第一行的 `Post` 可以和最后一行的一起留在 map 里等着建树，直到写出 JSON 为止，字符串一次都没有被拷贝。字段用 `Cow` 而不是 `&str`，是因为在写事务里页可能被改动，那时字段会退化成 `Cow::Owned` 的拷贝。rusqlite 那边不是不能借用，`row.get_ref` 就能借出 `&str`，但它只活到下一次 `next`；建树要等所有行到齐，借用 `row` 的 `Post` 活不过下一次 `next`，所以放进 map 的只能是拷贝。 + +这个优势有自己的适用范围。短行、逐行处理的负载两边差不多，SQLite 还略快；写事务里字符串照样要拷贝；TPC-C 这类负载看不出变化。聚合算子把借用的行留在手里，对应的页就要留在内存里，但这些页是 SQLite 读同样数据时也要进页缓存的干净文件页，内核可以随时回收；回收不掉的堆内存反而是零拷贝这边更少（表 3）。借用的结果活多久，只读事务就要开多久，而 LMDB 的旧页在有读者引用时不能回收，长时间持有结果又伴随大量写入，数据库文件会膨胀。 + +感谢阅读。文中的实现和测试代码都在 KiteSQL 仓库里：。 + +## 参考 + +[1] Athinagoras Skiadopoulos, Qian Li, Peter Kraft, Kostis Kaffes, Daniel Hong, Shana Mathew, David Bestor, Michael Cafarella, Vijay Gadepally, Goetz Graefe, Jeremy Kepner, Christos Kozyrakis, Tim Kraska, Michael Stonebraker, Lalith Suresh, Matei Zaharia. *DBOS: A DBMS-oriented Operating System*. PVLDB, 15(1): 21–30, 2022. doi:[10.14778/3485450.3485454](https://doi.org/10.14778/3485450.3485454). PDF: + +[2] GraphQL Specification, September 2025 Edition. GraphQL Foundation. . 项目主页： + +[3] Andrew Crotty, Viktor Leis, Andrew Pavlo. *Are You Sure You Want to Use MMAP in Your Database Management System?* CIDR 2022. 项目页：. PDF: 。候选标题见 Andy Pavlo 的讲座录像 00:52 处。 + +[4] LMDB API 文档，Symas Corporation：。`MDB_val` 一节说明返回的值"valid only until a subsequent update operation, or the end of the transaction"，并且"commonly point into the database itself"；`MDB_WRITEMAP` 一节说明默认的只读映射能防止野指针写坏数据库。 + +[5] PingCAP. *TiDB Computing*，"Mapping table data to Key-Value" 一节： + +[6] `examples/alloc_trace.rs`：。运行：`cargo run --release --example alloc_trace --features lmdb,orm` + +[7] `examples/borrowed_parse.rs`：。运行：`cargo run --release --example borrowed_parse --features lmdb,orm`