diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a79a7201..de7ad0aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,9 @@ jobs: - name: Run cargo test run: make test + + - name: Run cargo test through the async path + run: make test-async # 2 fmt: name: Rust fmt @@ -64,6 +67,9 @@ jobs: - name: Run sqllogictest suite run: make test-slt + + - name: Run sqllogictest suite through the async path + run: make test-slt-async # 3.1 sqlite-slt: name: SQLite sqllogictest corpus diff --git a/Cargo.lock b/Cargo.lock index 4797ae5f..93643157 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,6 +50,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "aligned-vec" version = "0.6.4" @@ -59,6 +65,12 @@ dependencies = [ "equator", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -154,12 +166,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + [[package]] name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -171,12 +201,67 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "asyncband" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2050892d4691cf110586070786712b504985a4d37ed64a9cd8a402718e1706" + +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.75" @@ -192,13 +277,28 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -206,7 +306,7 @@ dependencies = [ "quote", "regex", "rustc-hash", - "shlex", + "shlex 1.3.0", "syn 2.0.104", ] @@ -218,9 +318,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bitvec" @@ -243,6 +343,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "borsh" version = "1.5.7" @@ -311,6 +420,9 @@ name = "bytes" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +dependencies = [ + "serde", +] [[package]] name = "bzip2-sys" @@ -330,13 +442,14 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.29" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" +checksum = "4ff2a8126fff0f8c839080d619675a585e61720bb12d2baceae45769fd66f155" dependencies = [ + "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", ] [[package]] @@ -366,6 +479,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.41" @@ -376,8 +500,9 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -446,7 +571,7 @@ version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.104", @@ -467,12 +592,31 @@ dependencies = [ "error-code", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "7.1.4" @@ -483,6 +627,15 @@ dependencies = [ "unicode-width 0.2.1", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console" version = "0.15.11" @@ -496,12 +649,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_affinity" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342" +dependencies = [ + "libc", + "num_cpus", + "winapi", +] + [[package]] name = "cpp_demangle" version = "0.4.4" @@ -511,6 +685,34 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + +[[package]] +name = "crc32fast" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.5.1" @@ -566,6 +768,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -588,6 +800,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.3.1" @@ -644,6 +865,12 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + [[package]] name = "debugid" version = "0.8.0" @@ -653,14 +880,66 @@ dependencies = [ "uuid", ] +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.6", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "duration-str" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88959de2d447fd3eddcf1909d1f19fe084e27a056a6904203dc5d8b9e771c1e" +dependencies = [ + "rust_decimal", + "serde", + "thiserror 2.0.12", + "time", + "winnow 0.6.26", ] [[package]] @@ -755,12 +1034,76 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fail-parallel" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c29b33a0187823f1fa88b36980227dc96c7504ede2288e7d2a77d9d6d88b260c" +dependencies = [ + "log", + "once_cell", + "rand 0.9.5", + "tokio", +] + +[[package]] +name = "fastant" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e825441bfb2d831c47c97d05821552db8832479f44c571b97fededbf0099c07" +dependencies = [ + "small_ctor", + "web-time", +] + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "pear", + "serde", + "serde_json", + "serde_yaml", + "toml", + "uncased", + "version_check", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + [[package]] name = "findshlibs" version = "0.10.2" @@ -773,6 +1116,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.13.2", + "rustc_version", +] + [[package]] name = "fnv" version = "1.0.7" @@ -780,48 +1133,193 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "fs-err" -version = "3.3.2" +name = "foldhash" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c95b673b8f6f7235229ae11c5642d81b04c2e64c1e2fb417bc0cf73ca45f29" -dependencies = [ - "autocfg", -] +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] -name = "funty" -version = "2.0.0" +name = "form_urlencoded" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] [[package]] -name = "futures" -version = "0.3.31" +name = "foyer" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "6a59f42276891c0a4ce683fcda1aa1b4fd1065d68116c264e4bf49b9d318a0ed" dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", + "anyhow", + "asyncband", + "equivalent", + "foyer-common", + "foyer-memory", + "foyer-storage", + "foyer-tokio", "futures-util", + "mixtrics", + "pin-project", + "serde", + "tracing", ] [[package]] -name = "futures-channel" -version = "0.3.31" +name = "foyer-common" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" dependencies = [ - "futures-core", - "futures-sink", + "anyhow", + "bincode", + "bytes", + "cfg-if", + "foyer-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "serde", + "twox-hash", ] [[package]] -name = "futures-core" -version = "0.3.31" +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +dependencies = [ + "anyhow", + "asyncband", + "bitflags 2.13.2", + "datasketches", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "foyer-tokio", + "futures-util", + "hashbrown 0.17.1", + "itertools 0.15.0", + "mixtrics", + "parking_lot", + "paste", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-storage" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "118192f532ba013f0cdc607efc22324b9ed855baed185608af0daa986e835c6b" +dependencies = [ + "allocator-api2", + "anyhow", + "asyncband", + "bytes", + "core_affinity", + "equivalent", + "fastant", + "foyer-common", + "foyer-memory", + "foyer-tokio", + "fs4", + "futures-core", + "futures-util", + "hashbrown 0.17.1", + "io-uring", + "itertools 0.15.0", + "libc", + "lz4", + "parking_lot", + "pin-project", + "rand 0.10.3", + "serde", + "tracing", + "twox-hash", + "zstd", +] + +[[package]] +name = "foyer-tokio" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +dependencies = [ + "tokio", +] + +[[package]] +name = "fs-err" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c95b673b8f6f7235229ae11c5642d81b04c2e64c1e2fb417bc0cf73ca45f29" +dependencies = [ + "autocfg", +] + +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" @@ -900,8 +1398,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -912,10 +1412,24 @@ checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasi 0.14.2+wasi-0.2.4", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + [[package]] name = "gimli" version = "0.31.1" @@ -928,6 +1442,37 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.6.0" @@ -953,6 +1498,17 @@ version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hdrhistogram" version = "7.5.4" @@ -963,6 +1519,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -975,12 +1537,126 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "humantime" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.63" @@ -1005,12 +1681,116 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.10.0" @@ -1056,11 +1836,34 @@ dependencies = [ "log", "num-format", "once_cell", - "quick-xml", + "quick-xml 0.26.0", "rgb", "str_stack", ] +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + +[[package]] +name = "io-uring" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is-terminal" version = "0.4.16" @@ -1097,11 +1900,69 @@ dependencies = [ ] [[package]] -name = "itoa" -version = "1.0.15" +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.12", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.104", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.104", +] + [[package]] name = "jobserver" version = "0.1.33" @@ -1114,23 +1975,25 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] [[package]] name = "kite_sql" -version = "0.6.1" +version = "0.7.0" dependencies = [ "bumpalo", "chrono", "comfy-table", "criterion", "csv", + "foyer", "hashbrown 0.15.4", "indicatif", "js-sys", @@ -1146,9 +2009,11 @@ dependencies = [ "rustyline", "serde", "serde_json", + "slatedb", "sqlite", "sqlparser", "tempfile", + "tokio", "wasm-bindgen", "wasm-bindgen-test", ] @@ -1171,9 +2036,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -1234,6 +2099,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + [[package]] name = "lmdb" version = "0.8.0" @@ -1258,11 +2129,10 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] @@ -1272,6 +2142,40 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "macros-test" version = "0.4.0" @@ -1290,7 +2194,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] @@ -1342,6 +2256,27 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + +[[package]] +name = "mixtrics" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd94ebec15e61a36cd63b1e8816c83434e42926962cd21ee6e4db898d4d4f05" +dependencies = [ + "itertools 0.15.0", + "parking_lot", +] + [[package]] name = "nix" version = "0.26.4" @@ -1359,12 +2294,24 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases 0.1.1", "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases 0.2.1", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -1375,6 +2322,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1384,6 +2340,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-format" version = "0.4.4" @@ -1404,12 +2366,41 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.36.7" @@ -1419,6 +2410,48 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64", + "bytes", + "chrono", + "crc-fast", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools 0.15.0", + "md-5 0.11.0", + "nix 0.31.3", + "parking_lot", + "percent-encoding", + "quick-xml 0.41.0", + "rand 0.10.3", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.12", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", + "windows-sys 0.61.2", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -1437,6 +2470,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openssl-probe" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" + [[package]] name = "ordered-float" version = "4.6.0" @@ -1446,18 +2485,120 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.104", +] + [[package]] name = "owo-colors" version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link 0.2.1", +] + [[package]] name = "paste" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1510,6 +2651,21 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "pprof" version = "0.15.0" @@ -1530,7 +2686,7 @@ dependencies = [ "spin", "symbolic-demangle", "tempfile", - "thiserror", + "thiserror 2.0.12", ] [[package]] @@ -1560,6 +2716,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", + "version_check", + "yansi", +] + [[package]] name = "ptr_meta" version = "0.1.4" @@ -1636,7 +2805,7 @@ version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "pyo3-build-config", "quote", @@ -1653,35 +2822,129 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.40" +name = "quick-xml" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ - "proc-macro2", + "memchr", + "serde", ] [[package]] -name = "r-efi" -version = "5.3.0" +name = "quinn" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases 0.2.1", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.12", + "tokio", + "tracing", + "web-time", +] [[package]] -name = "radium" -version = "0.7.0" +name = "quinn-proto" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.12", + "tinyvec", + "tracing", + "web-time", +] [[package]] -name = "rand" +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases 0.2.1", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -1691,7 +2954,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1703,6 +2976,39 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.3", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xoshiro" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "662effc7698e08ea324d3acccf8d9d7f7bf79b9785e270a174ea36e56900c91d" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rayon" version = "1.10.0" @@ -1723,6 +3029,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + [[package]] name = "regex" version = "1.11.1" @@ -1761,6 +3076,45 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "rgb" version = "0.8.52" @@ -1770,6 +3124,20 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rkyv" version = "0.7.45" @@ -1819,7 +3187,7 @@ dependencies = [ "borsh", "bytes", "num-traits", - "rand", + "rand 0.8.5", "rkyv", "serde", "serde_json", @@ -1837,6 +3205,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustflags" version = "0.1.7" @@ -1849,13 +3226,88 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", "windows-sys 0.60.2", ] +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.21" @@ -1868,7 +3320,7 @@ version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.13.2", "cfg-if", "clipboard-win", "libc", @@ -1896,6 +3348,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1908,44 +3369,144 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" -version = "1.0.219" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 3.0.6", ] [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ "itoa", "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", "ryu", "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -1958,18 +3519,147 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + [[package]] name = "slab" version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" +[[package]] +name = "slatedb" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "371b0832486277108a130202eb2ce404e5ac426edd1490b5b973ca2d561b6055" +dependencies = [ + "async-channel", + "async-trait", + "atomic", + "backon", + "bitflags 2.13.2", + "bytes", + "chrono", + "crc32fast", + "crossbeam-skiplist", + "dotenvy", + "duration-str", + "fail-parallel", + "figment", + "flatbuffers", + "foyer", + "futures", + "log", + "lru", + "object_store", + "ouroboros", + "parking_lot", + "rand 0.10.3", + "serde", + "serde_json", + "siphasher", + "slatedb-common", + "slatedb-mirror", + "slatedb-txn-obj", + "smallvec", + "sysinfo", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tracing", + "ulid", + "url", + "uuid", + "walkdir", +] + +[[package]] +name = "slatedb-common" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6758f235603647a9a689d827852222d475e2011b1ea32d62d87d0808881f63" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "log", + "lru", + "object_store", + "parking_lot", + "rand 0.10.3", + "rand_xoshiro", + "serde", + "thread_local", + "tokio", +] + +[[package]] +name = "slatedb-mirror" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36b72ff54ab4f3ec6014f9aa316c6953d5c1ea1606534418081d7de76c6cb66" +dependencies = [ + "async-trait", + "backon", + "bytes", + "chrono", + "futures", + "hex", + "log", + "md-5 0.11.0", + "object_store", + "parking_lot", + "serde", + "serde_json", + "slatedb-common", + "thiserror 1.0.69", + "tokio", + "tokio-util", +] + +[[package]] +name = "slatedb-txn-obj" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fca84a01520f0ed924133792ece98ac1768be50bb5785db2487c63cf17a12c5" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "log", + "object_store", + "parking_lot", + "slatedb-common", + "thiserror 1.0.69", +] + +[[package]] +name = "small_ctor" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" + [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "spin" version = "0.10.0" @@ -2023,14 +3713,14 @@ dependencies = [ "humantime", "itertools 0.13.0", "libtest-mimic", - "md-5", + "md-5 0.10.6", "owo-colors", - "rand", + "rand 0.8.5", "regex", "similar", "subst", "tempfile", - "thiserror", + "thiserror 2.0.12", "tracing", ] @@ -2038,11 +3728,15 @@ dependencies = [ name = "sqllogictest-test" version = "0.4.0" dependencies = [ + "async-trait", "clap", "glob", "kite_sql", + "kite_sql_serde_macros", + "slatedb", "sqllogictest", "tempfile", + "tokio", ] [[package]] @@ -2060,6 +3754,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "str_stack" version = "0.1.0" @@ -2082,6 +3782,12 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "symbolic-common" version = "12.15.5" @@ -2138,6 +3844,40 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "sysinfo" +version = "0.35.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3ffa3e4ff2b324a57f7aeb3c349656c7b127c3c189520251a648102a92496e" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + [[package]] name = "tap" version = "1.0.1" @@ -2163,67 +3903,246 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "hashbrown 0.15.4", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", ] [[package]] -name = "thiserror-impl" -version = "2.0.12" +name = "toml_edit" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow 0.7.12", ] [[package]] -name = "tinytemplate" -version = "1.2.1" +name = "toml_write" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] -name = "tinyvec" -version = "1.9.0" +name = "tower" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ - "tinyvec_macros", + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", ] [[package]] -name = "tinyvec_macros" -version = "0.1.1" +name = "tower-http" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] [[package]] -name = "toml_datetime" -version = "0.6.11" +name = "tower-layer" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] -name = "toml_edit" -version = "0.22.27" +name = "tower-service" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap", - "toml_datetime", - "winnow", -] +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tpcc" @@ -2231,13 +4150,15 @@ version = "0.1.1" dependencies = [ "chrono", "clap", + "futures", "hdrhistogram", "indicatif", "kite_sql", "pprof", - "rand", + "rand 0.8.5", "rust_decimal", "sqlite", + "tokio", ] [[package]] @@ -2246,6 +4167,7 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2271,12 +4193,44 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" + [[package]] name = "typenum" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand 0.9.5", + "serde", + "web-time", +] + +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-ident" version = "1.0.18" @@ -2307,19 +4261,60 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "utf_types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + [[package]] name = "uuid" -version = "1.17.0" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +checksum = "4a06f08a856b009bc7bbaf7d3e79808595d032dd19074c0c2ef26e49e03e6bdc" dependencies = [ + "getrandom 0.4.3", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -2345,6 +4340,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2362,12 +4366,11 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" dependencies = [ "cfg-if", - "once_cell", "rustversion", "wasm-bindgen-macro", "wasm-bindgen-shared", @@ -2375,22 +4378,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501" dependencies = [ - "cfg-if", "js-sys", - "once_cell", + "tokio", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2398,33 +4399,32 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.104", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-bindgen-test" -version = "0.3.56" +version = "0.3.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e90e66d265d3a1efc0e72a54809ab90b9c0c515915c67cdf658689d2c22c6c" +checksum = "5b4913c0ade07e01ab574b20d2741538f070efa12cb7a1a1e2d5df6ea41cf522" dependencies = [ - "async-trait", "cast", "js-sys", "libm", @@ -2437,24 +4437,44 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test-macro", + "wasm-bindgen-test-shared", ] [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.56" +version = "0.3.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7150335716dce6028bead2b848e72f47b45e7b9422f64cccdc23bedca89affc1" +checksum = "238071aa4c5b268e57d21018621a2e2403c29dd11c55dbdf790ed47fdb9241c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 3.0.6", +] + +[[package]] +name = "wasm-bindgen-test-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab365ce29ba0232367502b2894556ae77a9c214d33dbf453fdba7f586905918" + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" dependencies = [ "js-sys", "wasm-bindgen", @@ -2470,6 +4490,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2501,6 +4530,28 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.61.2" @@ -2509,11 +4560,22 @@ checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", - "windows-link", + "windows-link 0.1.3", "windows-result", "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link 0.1.3", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.0" @@ -2542,13 +4604,29 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core", + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -2557,7 +4635,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -2587,6 +4665,15 @@ dependencies = [ "windows-targets 0.53.2", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -2619,6 +4706,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -2715,6 +4811,15 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winnow" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.12" @@ -2730,9 +4835,15 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.13.2", ] +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + [[package]] name = "wyz" version = "0.5.1" @@ -2742,6 +4853,35 @@ dependencies = [ "tap", ] +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.26" @@ -2761,3 +4901,97 @@ dependencies = [ "quote", "syn 2.0.104", ] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb02c3132d251ab1606e9fc913a7da6c6ed08cc263df15edf082e1dc497d157" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c8ea04668f5de8d8d51051714ba560c9536519cadcdb8829bba4fe2ad159b03" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index e5de3f0b..e86e8995 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "kite_sql" -version = "0.6.1" +version = "0.7.0" edition = "2021" authors = ["Kould ", "Xwg "] description = "SQL as a Function for Rust" @@ -38,6 +38,14 @@ pprof = ["pprof/criterion", "pprof/flamegraph"] python = ["parser", "dep:pyo3"] shell = ["parser", "dep:comfy-table", "dep:rustyline"] spill = [] +# Compiles the storage traits, the executor and the database API as async, for storages whose +# reads are asynchronous. Excludes the sync storages (`rocksdb`, `lmdb`) and `python`, `shell` +# and `wasm`; pick it in the final application, not in a library. +async = [] +# SlateDB storage, on object storage; needs the async path and a tokio runtime. +slatedb = ["async", "dep:foyer", "dep:slatedb"] +# SlateDB on Amazon S3 or an S3-compatible store, configured with `S3Config`. +slatedb-s3 = ["slatedb", "slatedb/aws"] wasm = [ "dep:js-sys", "dep:wasm-bindgen", @@ -60,6 +68,10 @@ required-features = ["lmdb", "orm", "parser"] name = "borrowed_parse" required-features = ["lmdb", "orm", "parser"] +[[example]] +name = "async_s3" +required-features = ["slatedb-s3", "orm"] + [dependencies] bumpalo = { version = "3", default-features = false, features = ["collections"] } hashbrown = { version = "0.15", default-features = false } @@ -82,6 +94,7 @@ pprof = { version = "0.15", features = ["flamegraph", "criterion"] } criterion = { version = "0.5", features = ["html_reports"] } indicatif = { version = "0.17" } tempfile = { version = "3.10" } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } # Benchmark sqlite = { version = "0.34" } serde = { version = "1", features = ["derive"] } @@ -92,6 +105,9 @@ serde_json = { version = "1" } lmdb = { version = "0.8.0", optional = true } lmdb-sys = { version = "0.8.0", optional = true } rocksdb = { version = "0.25", optional = true, default-features = false, features = ["bindgen-runtime"] } +slatedb = { version = "0.18", optional = true, default-features = false, features = ["foyer"] } +# The disk tier of the SlateDB block cache; the version SlateDB builds its caches on. +foyer = { version = "0.22", optional = true } rustyline = { version = "14", default-features = false, optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/Makefile b/Makefile index 5056f3aa..abc2f1b3 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,10 @@ GRCOV ?= grcov WASM_PACK ?= wasm-pack SQLLOGIC_PATH ?= tests/slt/**/*.slt SQLITE_LOGIC_PATH ?= tests/kitesql-testing/data/sqlite/**/*.slt +# Every feature except `async`, which excludes the sync storages and bindings. +SYNC_FEATURES ?= time,copy,decimal,macros,orm,parser,rocksdb,unsafe_txdb_checkpoint,lmdb,pprof,python,shell,spill,wasm +# The async path with every feature it supports. +ASYNC_FEATURES ?= async,time,copy,decimal,macros,orm,parser,spill SQLITE_LOGIC_ARGS ?= PYO3_PYTHON ?= /usr/bin/python3.12 TPCC_MEASURE_TIME ?= 15 @@ -40,12 +44,20 @@ MIRI_FEATURES ?= spill,decimal,lmdb MIRI_TESTS ?= types::tuple:: execution::test:: COVERAGE_REPORT_ARGS ?= --llvm --ignore-not-existing --keep-only 'src/**' --ignore 'src/**/tests/**' --ignore 'tests/**' --ignore 'tpcc/**' --excl-start 'GRCOV_EXCL_START' --excl-stop 'GRCOV_EXCL_STOP' -.PHONY: test test-python test-wasm test-slt test-all codecov codecov-html wasm-build check tpcc tpcc-kitesql-rocksdb tpcc-kitesql-lmdb tpcc-lmdb-flamegraph tpcc-lmdb-heaptrack tpcc-sqlite tpcc-dual cargo-check build wasm-examples native-examples fmt clippy fuzz miri +.PHONY: test test-async test-slatedb test-python test-wasm test-slt test-slt-async test-all codecov codecov-html wasm-build check tpcc tpcc-kitesql-rocksdb tpcc-kitesql-lmdb tpcc-lmdb-flamegraph tpcc-lmdb-heaptrack tpcc-sqlite tpcc-dual cargo-check build wasm-examples native-examples fmt clippy fuzz miri ## Run default Rust tests in the current environment (non-WASM). test: $(CARGO) test --all --features spill +## Run the same unit tests through the async path (`async` feature, SlateDB on an in-memory object store). +test-async: + $(CARGO) test -p kite_sql --lib --no-default-features --features $(ASYNC_FEATURES),slatedb + +## Run the SlateDB storage unit tests (async path, in-memory object store, S3 config). +test-slatedb: + $(CARGO) test -p kite_sql --lib --no-default-features --features $(ASYNC_FEATURES),slatedb-s3 storage::slatedb + ## Run Python binding API tests implemented with pyo3. test-python: PYO3_PYTHON=$(PYO3_PYTHON) $(CARGO) test --features python,decimal test_python_ @@ -70,6 +82,10 @@ test-wasm: test-slt: $(CARGO) run -p sqllogictest-test -- --path '$(SQLLOGIC_PATH)' +## Run the sqllogictest suite through the async path, over SlateDB (in-memory object store). +test-slt-async: + $(CARGO) run -p sqllogictest-test --no-default-features --features async -- --path '$(SQLLOGIC_PATH)' + .PHONY: test-slt-sqlite ## Run the external SQLite corpus (select1–5 excluded unless --include-select is passed). @@ -77,7 +93,7 @@ test-slt-sqlite: $(CARGO) run --release -p sqllogictest-test --bin sqlite -- --path '$(SQLITE_LOGIC_PATH)' $(SQLITE_LOGIC_ARGS) ## Convenience target to run every suite in sequence. -test-all: test test-wasm test-slt test-python +test-all: test test-async test-slatedb test-wasm test-slt test-slt-async test-python ## Generate an lcov coverage report for Codecov upload. codecov: @@ -121,9 +137,13 @@ miri: fmt: $(CARGO) fmt --all -- --check -## Execute clippy across all targets/features with warnings elevated to errors. +## Execute clippy across all targets/features with warnings elevated to errors, once for each +## path: `async` cannot be combined with the sync storages. clippy: - $(CARGO) clippy --all-targets --all-features -- -D warnings + $(CARGO) clippy --all-targets --features $(SYNC_FEATURES) -- -D warnings + $(CARGO) clippy --no-default-features --features $(ASYNC_FEATURES) -- -D warnings + $(CARGO) clippy --all-targets --no-default-features --features $(ASYNC_FEATURES),slatedb-s3 -- -D warnings + $(CARGO) clippy -p sqllogictest-test --all-targets --no-default-features --features async -- -D warnings ## Run formatting (check mode) and clippy linting together. check: fmt clippy diff --git a/README.md b/README.md index f96bbdec..a3d5bad5 100755 --- a/README.md +++ b/README.md @@ -32,7 +32,9 @@ **KiteSQL** is a lightweight embedded relational database written in Rust, inspired by **MyRocks** and **SQLite**. It runs inside your application with no external service: execute SQL directly, or use typed ORM models, migrations, and builder-style queries. - Most of the SQL 2016 syntax -- All metadata and data stored in KV storage (RocksDB, LMDB, or in-memory) +- All metadata and data stored in KV storage: RocksDB, LMDB, in-memory, or [SlateDB](https://slatedb.io) on S3 +- **Zero-copy reads**: on LMDB, strings go from the mmap'd file into your structs as `Cow::Borrowed`, without a copy ([how](https://kitesql.kould2333.org/blog/zero-copy-sql-on-lmdb.html)) +- **Async API on object storage**: the same engine as async, on SlateDB over Amazon S3 or an S3-compatible store such as MinIO or R2, see [Async API on S3](docs/features.md#async-api-on-s3-features--slatedb-s3) - Typed ORM with schema migration (`orm` feature), see [`src/orm/README.md`](src/orm/README.md) - WebAssembly and Python bindings @@ -87,7 +89,7 @@ fn main() -> Result<(), DatabaseError> { } ``` -More: [hello_world](examples/hello_world.rs), [transaction](examples/transaction.rs), [program](examples/program.rs). +More: [hello_world](examples/hello_world.rs), [transaction](examples/transaction.rs), [program](examples/program.rs), [borrowed_parse](examples/borrowed_parse.rs) (zero-copy reads against SQLite). ## Storage Backends | Builder | Storage | @@ -96,6 +98,7 @@ More: [hello_world](examples/hello_world.rs), [transaction](examples/transaction | `build_lmdb()` | LMDB (feature `lmdb`), stronger for read-heavy workloads | | `build_in_memory()` | In-memory, for tests and temporary data | | `build_optimistic()` | RocksDB with optimistic transactions | +| `build_with_storage(SlateDbStorage::open_s3(..))` | SlateDB on S3 (feature `slatedb-s3`, async API), see [Async API on S3](docs/features.md#async-api-on-s3-features--slatedb-s3) | Feature flags, checkpoints, and transaction isolation: [docs/features.md](docs/features.md). @@ -133,13 +136,25 @@ print(list(db.run("select * from demo"))) ## TPC-C `make tpcc` runs the benchmark (`--backend kitesql-lmdb|kitesql-rocksdb|sqlite`); `make tpcc-dual` cross-checks every statement against SQLite. -720-second single-thread run on an i9-13900HX (32 GB, KIOXIA EXCERIA PLUS G3), every backend pinned to the same P-core and run without fsync on commit. Latencies are p90 in µs, including commit. +720-second runs on an i9-13900HX (32 GB, KIOXIA EXCERIA PLUS G3), without fsync on commit. Latencies are p90 in µs, including commit. SlateDB runs on a local S3 server ([s3s-fs](https://github.com/Nugine/s3s)) with its block cache in memory and on local disk. + +1 thread, 1 warehouse, every backend pinned to the same P-core: + +| Backend | TpmC | New-Order | Payment | Order-Status | Delivery | Stock-Level | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| KiteSQL LMDB | 254023 | 211 | 38 | 25 | 293 | 175 | +| KiteSQL RocksDB | 47488 | 372 | 85 | 415 | 15471 | 389 | +| KiteSQL SlateDB (S3) | 12016 | 677 | 240 | 399 | 74047 | 1638 | +| SQLite (WAL, `synchronous=OFF`) | 214001 | 251 | 37 | 32 | 338 | 227 | + +4 threads, 4 warehouses (one home warehouse per terminal), pinned to four P-cores: | Backend | TpmC | New-Order | Payment | Order-Status | Delivery | Stock-Level | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| KiteSQL LMDB | 225720 | 240 | 44 | 25 | 326 | 179 | -| KiteSQL RocksDB | 48746 | 343 | 78 | 140 | 14351 | 587 | -| SQLite (WAL, `synchronous=OFF`) | 192110 | 270 | 40 | 33 | 342 | 231 | +| KiteSQL LMDB | 240584 | 930 | 780 | 29 | 890 | 207 | +| KiteSQL RocksDB | 212936 | 505 | 109 | 285 | 8991 | 400 | +| KiteSQL SlateDB (S3) | 19010 | 1749 | 1816 | 2481 | 66495 | 2523 | +| SQLite (WAL, `synchronous=OFF`) | 205896 | 254 | 41 | 45 | 336 | 380 | 👉 [Details and how to reproduce](tpcc/README.md) diff --git a/docs/features.md b/docs/features.md index d03b726f..54765a05 100644 --- a/docs/features.md +++ b/docs/features.md @@ -42,6 +42,67 @@ See [the ORM guide](../src/orm/README.md) for the full ORM guide, including: - public ORM structs, enums, and traits - related `ResultIter::orm::()` integration +### Async API on S3: `features = ["slatedb-s3"]` +With the `async` feature, the storage traits, the executor and the database API compile as async from the same source; the sync build is unchanged. The `slatedb-s3` feature adds `SlateDbStorage`, which keeps the data in an S3 bucket through [SlateDB](https://slatedb.io), an LSM tree on object storage, and runs on tokio. Any process that opens the bucket path sees the same database. + +```rust +use kite_sql::db::{AsyncIter, DataBaseBuilder}; +use kite_sql::storage::slatedb::{S3Config, SlateDbStorage}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Credentials and region come from the `AWS_*` environment variables. + let config = S3Config::new("my-bucket"); + let storage = SlateDbStorage::open_s3("kitesql/demo", &config, Default::default()).await?; + let mut database = DataBaseBuilder::path(".").build_with_storage(storage.clone())?; + + database.migrate::().await?; + database.insert(&User { id: 1, name: "Alice".to_string(), age: Some(18) }).await?; + let mut users = database.fetch::().await?; + while let Some(user) = users.next().await { + println!("{:?}", user?); + } + users.done().await?; + + drop(database); + // Flushes what is still in memory to S3. + storage.close().await?; + Ok(()) +} +``` + +`async` excludes the sync storages (`rocksdb`, `lmdb`) and the shell, Python and Wasm bindings: pick one path per build. The full example is [async_s3](../examples/async_s3.rs): +```bash +KITESQL_S3_BUCKET=my-bucket AWS_REGION=us-east-1 AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... \ +cargo run --example async_s3 --no-default-features --features slatedb-s3,orm +``` + +A reopened database loads its tables with `database.load(CatalogKind::Table(name))` before +querying them, as `async_s3` does. + +By default a commit waits until its write is durable in S3. `storage.await_durable(false)` makes +it return once the write is applied in memory; a crash may then lose the latest commits. + +Reads that miss the block cache cost an S3 request each, so the cache size decides how fast +queries run. `SlateDbCacheOptions` sizes the memory cache and adds a local disk tier: +```rust +use kite_sql::storage::slatedb::{S3Config, SlateDbCacheOptions, SlateDbDiskCache, SlateDbStorage}; + +let cache = SlateDbCacheOptions { + memory_bytes: Some(4 << 30), + disk: Some(SlateDbDiskCache { + dir: "./slatedb-cache".into(), + // The cache files grow as they fill, so a large capacity costs nothing up front. + capacity_bytes: 64 << 30, + }), + // Keep the blocks compactions write, which otherwise leave the cache. + cache_compaction_output: true, +}; +let storage = SlateDbStorage::open_s3("kitesql/demo", &S3Config::new("my-bucket"), cache).await?; +``` +`SlateDbScanOptions` sets how scans read SST blocks: 256KiB per request, 4 requests in flight and +cached blocks by default. + ### Spill-backed Aggregation: `features = ["spill"]` The native-only `spill` feature bounds memory usage for large grouped @@ -132,7 +193,7 @@ table_function!(MyTableFunction::test_numbers(LogicalType::Integer) -> [c1: Logi DataValue::Int32(Some(i)), DataValue::Int32(Some(i)), ] - }))) as Box>>) + }))) as TableFunctionIter) })); let kite_sql = DataBaseBuilder::path("./data") .register_table_function(MyTableFunction::new()) diff --git a/examples/async_s3.rs b/examples/async_s3.rs new file mode 100644 index 00000000..b43e2dee --- /dev/null +++ b/examples/async_s3.rs @@ -0,0 +1,121 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The async API on Amazon S3, through SlateDB, run on tokio. It writes a table on a spawned +//! task, closes the database, opens it again from the bucket and reads the table back. +//! +//! ```text +//! KITESQL_S3_BUCKET=my-bucket AWS_REGION=us-east-1 \ +//! AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... \ +//! cargo run --example async_s3 --no-default-features --features slatedb-s3,orm +//! ``` +//! +//! Credentials and region come from the standard `AWS_*` environment variables or the default +//! AWS chain. For an S3-compatible store such as MinIO, also set `KITESQL_S3_ENDPOINT` (and +//! `KITESQL_S3_ALLOW_HTTP=true` for a plain `http://` endpoint). The data goes under the +//! `KITESQL_S3_PATH` prefix, `kitesql/async_s3` by default. + +#[cfg(all(not(target_arch = "wasm32"), feature = "slatedb-s3", feature = "orm"))] +mod app { + use kite_sql::db::{AsyncIter, CatalogKind, DataBaseBuilder}; + use kite_sql::storage::slatedb::{S3Config, SlateDbStorage}; + use kite_sql::Model; + use std::borrow::Cow; + use std::env; + use std::error::Error; + + #[derive(Default, Debug, PartialEq, Model)] + #[model(table = "my_struct")] + pub struct MyStruct<'a> { + #[model(primary_key)] + pub c1: i32, + pub c2: Cow<'a, str>, + } + + fn config() -> Result> { + let bucket = env::var("KITESQL_S3_BUCKET") + .map_err(|_| "set KITESQL_S3_BUCKET to the bucket to store the database in")?; + Ok(S3Config { + endpoint: env::var("KITESQL_S3_ENDPOINT").ok(), + allow_http: env::var("KITESQL_S3_ALLOW_HTTP").is_ok_and(|value| value == "true"), + ..S3Config::new(bucket) + }) + } + + pub async fn run() -> Result<(), Box> { + let config = config()?; + let path = env::var("KITESQL_S3_PATH").unwrap_or_else(|_| "kitesql/async_s3".to_string()); + println!("opening {path} with {config:?}"); + + let storage = SlateDbStorage::open_s3(path.as_str(), &config, Default::default()).await?; + let mut database = DataBaseBuilder::path(".").build_with_storage(storage.clone())?; + // The database and its futures are `Send`, so the work can run on any worker thread. + tokio::spawn(async move { + database.drop_table_if_exists::().await?; + database.create_table_if_not_exists::().await?; + for (c1, c2) in [(0, "zero"), (1, "one"), (2, "two")] { + database.insert(&MyStruct { c1, c2: c2.into() }).await?; + } + database + .bind(|ctx| { + ctx.mutate::()? + .filter(|e| e.column(MyStruct::c1())?.eq(1))? + .update(|u| u.set_value(MyStruct::c2(), "ONE")) + }) + .await? + .done() + .await?; + database + .bind(|ctx| { + ctx.mutate::()? + .filter(|e| e.column(MyStruct::c1())?.eq(2))? + .delete() + }) + .await? + .done() + .await + }) + .await + .expect("example task panicked")?; + // Flushes what is still in memory to S3 and releases the database. + storage.close().await?; + + // A new process, or another machine, can open the same data from the bucket. + let storage = SlateDbStorage::open_s3(path.as_str(), &config, Default::default()).await?; + let mut database = DataBaseBuilder::path(".").build_with_storage(storage.clone())?; + database + .load(CatalogKind::Table("my_struct".to_string())) + .await?; + let mut count = 0; + let mut rows = database.fetch::().await?; + while let Some(row) = rows.next().await { + println!("{:?}", row?); + count += 1; + } + rows.done().await?; + println!("row count = {count}"); + drop(database); + storage.close().await?; + Ok(()) + } +} + +#[cfg(all(not(target_arch = "wasm32"), feature = "slatedb-s3", feature = "orm"))] +#[tokio::main] +async fn main() -> Result<(), Box> { + app::run().await +} + +#[cfg(not(all(not(target_arch = "wasm32"), feature = "slatedb-s3", feature = "orm")))] +fn main() {} diff --git a/examples/hello_world.rs b/examples/hello_world.rs index 05bfab8e..cc37a282 100644 --- a/examples/hello_world.rs +++ b/examples/hello_world.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(all(not(target_arch = "wasm32"), feature = "orm"))] +#[cfg(all(not(target_arch = "wasm32"), not(feature = "async"), feature = "orm"))] mod app { use kite_sql::db::{DataBaseBuilder, Database}; use kite_sql::errors::DatabaseError; @@ -129,13 +129,18 @@ mod app { } } -#[cfg(target_arch = "wasm32")] +// The examples use the sync API. +#[cfg(any(target_arch = "wasm32", feature = "async"))] fn main() {} -#[cfg(all(not(target_arch = "wasm32"), feature = "orm"))] +#[cfg(all(not(target_arch = "wasm32"), not(feature = "async"), feature = "orm"))] fn main() -> Result<(), kite_sql::errors::DatabaseError> { app::run() } -#[cfg(all(not(target_arch = "wasm32"), not(feature = "orm")))] +#[cfg(all( + not(target_arch = "wasm32"), + not(feature = "async"), + not(feature = "orm") +))] fn main() {} diff --git a/examples/program.rs b/examples/program.rs index 312ffd28..d5f52d12 100644 --- a/examples/program.rs +++ b/examples/program.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(any(target_arch = "wasm32", feature = "async")))] mod app { use kite_sql::db::DataBaseBuilder; use kite_sql::errors::DatabaseError; @@ -94,10 +94,11 @@ mod app { } } -#[cfg(target_arch = "wasm32")] +// The examples use the sync API. +#[cfg(any(target_arch = "wasm32", feature = "async"))] fn main() {} -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(any(target_arch = "wasm32", feature = "async")))] fn main() -> Result<(), kite_sql::errors::DatabaseError> { app::run() } diff --git a/examples/transaction.rs b/examples/transaction.rs index 28c32f61..ecb90899 100644 --- a/examples/transaction.rs +++ b/examples/transaction.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(any(target_arch = "wasm32", feature = "async")))] mod app { use kite_sql::db::DataBaseBuilder; use kite_sql::errors::DatabaseError; @@ -82,10 +82,11 @@ mod app { } } -#[cfg(target_arch = "wasm32")] +// The examples use the sync API. +#[cfg(any(target_arch = "wasm32", feature = "async"))] fn main() {} -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(any(target_arch = "wasm32", feature = "async")))] fn main() -> Result<(), kite_sql::errors::DatabaseError> { app::run() } diff --git a/kite_sql_serde_macros/Cargo.toml b/kite_sql_serde_macros/Cargo.toml index dba525d4..57891644 100644 --- a/kite_sql_serde_macros/Cargo.toml +++ b/kite_sql_serde_macros/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/KipData/KiteSQL" darling = "0.20" proc-macro2 = "1" quote = "1" -syn = "2" +syn = { version = "2", features = ["full", "visit-mut"] } [lib] path = "src/lib.rs" diff --git a/kite_sql_serde_macros/src/lib.rs b/kite_sql_serde_macros/src/lib.rs index 29a65654..772be228 100644 --- a/kite_sql_serde_macros/src/lib.rs +++ b/kite_sql_serde_macros/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod maybe_async; mod orm; mod projection; mod reference_serialization; @@ -52,3 +53,19 @@ pub fn projection(input: TokenStream) -> TokenStream { Err(e) => e.to_compile_error().into(), } } + +/// Compiles one async source as the sync path, or as the async path with the crate's `async` +/// feature, in place and under the same names. +/// +/// ```ignore +/// #[maybe_async] +/// impl Scan { +/// async fn count(&self, txn: &T) -> Result { .. } +/// } +/// ``` +/// +/// `#[sync_only]` / `#[async_only]` keep an impl item, trait item or nested item in one path only. +#[proc_macro_attribute] +pub fn maybe_async(attr: TokenStream, item: TokenStream) -> TokenStream { + maybe_async::handle(attr.into(), item.into()).into() +} diff --git a/kite_sql_serde_macros/src/maybe_async.rs b/kite_sql_serde_macros/src/maybe_async.rs new file mode 100644 index 00000000..417c413d --- /dev/null +++ b/kite_sql_serde_macros/src/maybe_async.rs @@ -0,0 +1,297 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! `#[maybe_async]` compiles one async source as the sync path (by default) or the async path (with +//! the crate's `async` feature), in place and under the same names. It emits both versions of +//! the item, each under its `cfg`, so only one ever exists in a build. +//! +//! - **sync**: `async` removed from signatures, `.await` removed from expressions (macro +//! arguments included), `async |..|` +//! closures lose `async`, `AsyncFn*` bounds become `Fn*`, and an `async { .. }` block becomes +//! an immediately called closure `(|| { .. })()`, so `?` and `return` inside it keep leaving +//! just the block. This is the code the sync backends run, identical to hand-written sync code. +//! - **async**: the code as written, except that `async fn` in trait definitions becomes +//! `fn .. -> impl Future + Send`, so generic code can prove its futures are `Send`, +//! and an `async fn` marked `#[test]` becomes a plain test running its body on +//! `crate::maybe_async::block_on`, so one test source covers both paths. +//! +//! `#[sync_only]` / `#[async_only]` keep an impl item, trait item or nested item in one path +//! only. `#[keep_async_fn]` on a trait keeps its `async fn`s as written on the async path; use it +//! for traits with concrete implementors only, where callers see the concrete future type anyway. + +use proc_macro2::{Group, TokenStream, TokenTree}; +use quote::quote; +use syn::visit_mut::{self, VisitMut}; +use syn::{Attribute, Expr, Ident, ImplItem, Item, ReturnType, Signature, TraitItem, Type}; + +#[derive(Clone, Copy, PartialEq)] +enum Variant { + Sync, + Async, +} + +impl Variant { + fn keep_attr(self) -> &'static str { + match self { + Variant::Sync => "sync_only", + Variant::Async => "async_only", + } + } + + fn drop_attr(self) -> &'static str { + match self { + Variant::Sync => "async_only", + Variant::Async => "sync_only", + } + } +} + +struct Transform { + variant: Variant, +} + +impl Transform { + /// Returns `false` if the item belongs to the other copy; strips the marker attributes. + fn retain(&self, attrs: &mut Vec) -> bool { + if attrs + .iter() + .any(|attr| attr.path().is_ident(self.variant.drop_attr())) + { + return false; + } + attrs.retain(|attr| !attr.path().is_ident(self.variant.keep_attr())); + true + } + + fn retain_items(&self, items: &mut Vec) { + items.retain_mut(|item| match item_attrs(item) { + Some(attrs) => self.retain(attrs), + None => true, + }); + } +} + +impl VisitMut for Transform { + fn visit_file_mut(&mut self, file: &mut syn::File) { + self.retain_items(&mut file.items); + visit_mut::visit_file_mut(self, file); + } + + fn visit_item_mod_mut(&mut self, item: &mut syn::ItemMod) { + if let Some((_, items)) = &mut item.content { + self.retain_items(items); + } + visit_mut::visit_item_mod_mut(self, item); + } + + fn visit_item_impl_mut(&mut self, item: &mut syn::ItemImpl) { + item.items.retain_mut(|item| match item { + ImplItem::Fn(f) => self.retain(&mut f.attrs), + ImplItem::Const(c) => self.retain(&mut c.attrs), + ImplItem::Type(t) => self.retain(&mut t.attrs), + _ => true, + }); + visit_mut::visit_item_impl_mut(self, item); + } + + fn visit_item_trait_mut(&mut self, item: &mut syn::ItemTrait) { + item.items.retain_mut(|item| match item { + TraitItem::Fn(f) => self.retain(&mut f.attrs), + TraitItem::Const(c) => self.retain(&mut c.attrs), + TraitItem::Type(t) => self.retain(&mut t.attrs), + _ => true, + }); + let keep_async_fn = item + .attrs + .iter() + .any(|attr| attr.path().is_ident("keep_async_fn")); + item.attrs + .retain(|attr| !attr.path().is_ident("keep_async_fn")); + visit_mut::visit_item_trait_mut(self, item); + if self.variant == Variant::Async && keep_async_fn { + item.attrs + .push(syn::parse_quote!(#[allow(async_fn_in_trait)])); + } else if self.variant == Variant::Async { + for item in &mut item.items { + if let TraitItem::Fn(f) = item { + desugar_trait_async_fn(f); + } + } + } + } + + fn visit_item_fn_mut(&mut self, item: &mut syn::ItemFn) { + visit_mut::visit_item_fn_mut(self, item); + // An async test runs to completion on `crate::maybe_async::block_on`. + if self.variant == Variant::Async + && item.sig.asyncness.is_some() + && item.attrs.iter().any(|attr| attr.path().is_ident("test")) + { + item.sig.asyncness = None; + let block = &item.block; + *item.block = syn::parse_quote!({ crate::maybe_async::block_on(async move #block) }); + } + } + + fn visit_signature_mut(&mut self, sig: &mut Signature) { + if self.variant == Variant::Sync { + sig.asyncness = None; + } + visit_mut::visit_signature_mut(self, sig); + } + + fn visit_expr_mut(&mut self, expr: &mut Expr) { + if self.variant == Variant::Sync { + while let Expr::Await(await_expr) = expr { + *expr = (*await_expr.base).clone(); + } + if let Expr::Closure(closure) = expr { + closure.asyncness = None; + } + if let Expr::Async(block) = expr { + let capture = block.capture; + let body = block.block.clone(); + *expr = syn::parse_quote!((#capture || #body)()); + } + } + visit_mut::visit_expr_mut(self, expr); + } + + fn visit_macro_mut(&mut self, mac: &mut syn::Macro) { + // Macro arguments are opaque tokens, so `.await` and `async` are removed token by token. + if self.variant == Variant::Sync { + mac.tokens = strip_await(std::mem::take(&mut mac.tokens)); + } + visit_mut::visit_macro_mut(self, mac); + } + + fn visit_path_segment_mut(&mut self, segment: &mut syn::PathSegment) { + if self.variant == Variant::Sync { + let sync_name = match segment.ident.to_string().as_str() { + "AsyncFnOnce" => Some("FnOnce"), + "AsyncFnMut" => Some("FnMut"), + "AsyncFn" => Some("Fn"), + _ => None, + }; + if let Some(name) = sync_name { + segment.ident = Ident::new(name, segment.ident.span()); + } + } + visit_mut::visit_path_segment_mut(self, segment); + } +} + +/// Drops every `.await` from `tokens`, and `async` before `fn` and closures, inside groups too. +fn strip_await(tokens: TokenStream) -> TokenStream { + let tokens = tokens.into_iter().collect::>(); + let is_ident = |i: usize, name: &str| matches!(tokens.get(i), Some(TokenTree::Ident(ident)) if ident == name); + let is_punct = |i: usize, c: char| matches!(tokens.get(i), Some(TokenTree::Punct(punct)) if punct.as_char() == c); + let mut output = Vec::with_capacity(tokens.len()); + let mut i = 0; + while i < tokens.len() { + if is_punct(i, '.') && is_ident(i + 1, "await") { + i += 2; + continue; + } + if is_ident(i, "async") + && (is_ident(i + 1, "fn") + || is_punct(i + 1, '|') + || (is_ident(i + 1, "move") && is_punct(i + 2, '|'))) + { + i += 1; + continue; + } + output.push(match &tokens[i] { + TokenTree::Group(group) => { + let mut stripped = Group::new(group.delimiter(), strip_await(group.stream())); + stripped.set_span(group.span()); + TokenTree::Group(stripped) + } + token => token.clone(), + }); + i += 1; + } + output.into_iter().collect() +} + +/// `async fn f(..) -> R { body }` in a trait becomes +/// `fn f(..) -> impl Future + Send { async move { let r: R = { body }; r } }`. +/// Naming `R` on the binding lets `?` inside the body infer its error type. +fn desugar_trait_async_fn(f: &mut syn::TraitItemFn) { + if f.sig.asyncness.take().is_none() { + return; + } + let output: Type = match &f.sig.output { + ReturnType::Default => syn::parse_quote!(()), + ReturnType::Type(_, ty) => (**ty).clone(), + }; + f.sig.output = syn::parse_quote! { + -> impl ::core::future::Future + ::core::marker::Send + }; + if let Some(block) = &f.default { + // A body that only leaves through `return` makes the trailing binding unreachable. + f.attrs + .push(syn::parse_quote!(#[allow(unreachable_code, clippy::let_and_return)])); + f.default = Some(syn::parse_quote! {{ + async move { + let __maybe_async_output: #output = #block; + __maybe_async_output + } + }}); + } +} + +fn item_attrs(item: &mut Item) -> Option<&mut Vec> { + Some(match item { + Item::Const(i) => &mut i.attrs, + Item::Enum(i) => &mut i.attrs, + Item::Fn(i) => &mut i.attrs, + Item::Impl(i) => &mut i.attrs, + Item::Macro(i) => &mut i.attrs, + Item::Mod(i) => &mut i.attrs, + Item::Static(i) => &mut i.attrs, + Item::Struct(i) => &mut i.attrs, + Item::Trait(i) => &mut i.attrs, + Item::Type(i) => &mut i.attrs, + Item::Union(i) => &mut i.attrs, + Item::Use(i) => &mut i.attrs, + _ => return None, + }) +} + +pub(crate) fn handle(attr: TokenStream, item: TokenStream) -> TokenStream { + if !attr.is_empty() { + return syn::Error::new_spanned(attr, "`#[maybe_async]` takes no arguments") + .to_compile_error(); + } + let item: Item = match syn::parse2(item) { + Ok(item) => item, + Err(err) => return err.to_compile_error(), + }; + let copy = |variant| { + let mut file = syn::File { + shebang: None, + attrs: Vec::new(), + items: vec![item.clone()], + }; + Transform { variant }.visit_file_mut(&mut file); + file.items + }; + let sync = copy(Variant::Sync); + let async_copy = copy(Variant::Async); + quote! { + #(#[cfg(not(feature = "async"))] #sync)* + #(#[cfg(feature = "async")] #async_copy)* + } +} diff --git a/src/binder/aggregate.rs b/src/binder/aggregate.rs index c34afc20..59c1879d 100644 --- a/src/binder/aggregate.rs +++ b/src/binder/aggregate.rs @@ -416,6 +416,7 @@ impl ExprVisitorMut for AggregateOutputBinder<'_> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::AggregateOutputBinder; use crate::binder::test::build_t1_table; @@ -524,8 +525,9 @@ mod tests { } #[test] - fn test_validate_having_orderby_rejects_missing_group_expr() -> Result<(), DatabaseError> { - let tables = build_t1_table()?; + async fn test_validate_having_orderby_rejects_missing_group_expr() -> Result<(), DatabaseError> + { + let tables = build_t1_table().await?; let scala_functions = Default::default(); let table_functions = Default::default(); let transaction = tables.storage.transaction()?; diff --git a/src/binder/create_table.rs b/src/binder/create_table.rs index 4f46aa68..0450a25d 100644 --- a/src/binder/create_table.rs +++ b/src/binder/create_table.rs @@ -61,20 +61,18 @@ impl> Binder<'_, '_, T, // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::binder::BinderContext; use crate::catalog::ColumnDesc; - use crate::storage::rocksdb::RocksStorage; use crate::storage::Storage; use crate::types::CharLengthUnits; use crate::types::LogicalType; - use tempfile::TempDir; #[test] - fn test_create_bind() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_create_bind() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let names = crate::storage::NameMapping::default(); let table_cache = crate::storage::TableCache::default(); diff --git a/src/binder/distinct.rs b/src/binder/distinct.rs index ef58e38a..9099306c 100644 --- a/src/binder/distinct.rs +++ b/src/binder/distinct.rs @@ -126,6 +126,7 @@ impl ExprVisitorMut for DistinctOutputBinder<'_> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::DistinctOutputBinder; use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; diff --git a/src/binder/mod.rs b/src/binder/mod.rs index 62ddc6ca..2dd8eed8 100644 --- a/src/binder/mod.rs +++ b/src/binder/mod.rs @@ -799,17 +799,16 @@ pub(crate) fn is_valid_identifier(s: &str) -> bool { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] pub mod test { use crate::binder::{is_valid_identifier, Binder, BinderContext}; use crate::catalog::{ColumnCatalog, ColumnDesc, TableCatalog}; use crate::errors::DatabaseError; use crate::planner::{LogicalPlan, PlanArena, TableArenaCell}; - use crate::storage::rocksdb::RocksStorage; + use crate::storage::TestStorage; use crate::storage::{table_codec::TableCodec, Storage, StorageTransaction}; use crate::types::ColumnId; use crate::types::LogicalType::Integer; - use std::path::PathBuf; - use tempfile::TempDir; pub(crate) struct TableState { pub(crate) table: TableCatalog, @@ -855,10 +854,9 @@ pub mod test { } } - pub(crate) fn build_t1_table() -> Result, DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + pub(crate) async fn build_t1_table() -> Result, DatabaseError> { let table_arena = TableArenaCell::default(); - let storage = build_test_catalog(temp_dir.path(), &table_arena)?; + let storage = build_test_catalog(&table_arena).await?; let table = { let arena = table_arena.borrow(); arena.names.table(&arena.tables, "t1").unwrap().clone() @@ -871,70 +869,76 @@ pub mod test { }) } - pub(crate) fn build_test_catalog( - path: impl Into + Send, + pub(crate) async fn build_test_catalog( table_arena: &TableArenaCell, - ) -> Result { - let storage = RocksStorage::new(path)?; + ) -> Result { + let storage = crate::storage::new_test_storage().await; let mut transaction = storage.transaction()?; let mut table_codec = TableCodec::default(); let mut plan_arena = PlanArena::new(table_arena); - if let Some(table) = transaction.create_table( - &mut table_codec, - &mut plan_arena, - "t1".to_string(), - vec![ - ColumnCatalog::new( - "c1".to_string(), - false, - ColumnDesc::new(Integer, Some(0), false, None)?, - ), - ColumnCatalog::new( - "c2".to_string(), - false, - ColumnDesc::new(Integer, None, true, None)?, - ), - ], - false, - )? { + if let Some(table) = transaction + .create_table( + &mut table_codec, + &mut plan_arena, + "t1".to_string(), + vec![ + ColumnCatalog::new( + "c1".to_string(), + false, + ColumnDesc::new(Integer, Some(0), false, None)?, + ), + ColumnCatalog::new( + "c2".to_string(), + false, + ColumnDesc::new(Integer, None, true, None)?, + ), + ], + false, + ) + .await? + { let table = table.transplant_to_table_arena(&plan_arena)?; table_arena.borrow_mut().insert_table(table); } let mut plan_arena = PlanArena::new(table_arena); - if let Some(table) = transaction.create_table( - &mut table_codec, - &mut plan_arena, - "t2".to_string(), - vec![ - ColumnCatalog::new( - "c3".to_string(), - false, - ColumnDesc::new(Integer, Some(0), false, None)?, - ), - ColumnCatalog::new( - "c4".to_string(), - false, - ColumnDesc::new(Integer, None, false, None)?, - ), - ], - false, - )? { + if let Some(table) = transaction + .create_table( + &mut table_codec, + &mut plan_arena, + "t2".to_string(), + vec![ + ColumnCatalog::new( + "c3".to_string(), + false, + ColumnDesc::new(Integer, Some(0), false, None)?, + ), + ColumnCatalog::new( + "c4".to_string(), + false, + ColumnDesc::new(Integer, None, false, None)?, + ), + ], + false, + ) + .await? + { let table = table.transplant_to_table_arena(&plan_arena)?; table_arena.borrow_mut().insert_table(table); } - transaction.commit()?; + transaction.commit().await?; Ok(storage) } #[test] - fn sorted_bindings_preserve_names_positions_and_scope_isolation() -> Result<(), DatabaseError> { + async fn sorted_bindings_preserve_names_positions_and_scope_isolation( + ) -> Result<(), DatabaseError> { use crate::binder::Source; use crate::expression::ScalarExpression; - let state = build_t1_table()?; + let state = build_t1_table().await?; let arena = state.table_arena.borrow(); let plan_arena = PlanArena::new(&state.table_arena); let transaction = state.storage.transaction()?; diff --git a/src/binder/parser.rs b/src/binder/parser.rs index e63b74b5..6f10921a 100644 --- a/src/binder/parser.rs +++ b/src/binder/parser.rs @@ -22,7 +22,8 @@ use super::{is_valid_identifier, Binder, CteBinding, QueryBindStep, SetOperatorK use crate::binder::copy::{ExtSource, FileFormat}; use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef, TableName}; use crate::db::{ - BindSource, Database, DatabaseIter, Executor, State, Transaction, TransactionIter, + BindSource, ConsumeMut, Database, DatabaseIter, Executor, OwnedPtr, State, Transaction, + TransactionIter, }; use crate::errors::{DatabaseError, SqlErrorSpan}; use crate::expression; @@ -170,13 +171,14 @@ fn statement_mutates_catalog_or_statistics(statement: &Statement) -> Result Database { /// Parses SQL statements using this database's configured parser recursion limit. pub fn prepare_all>(&self, sql: T) -> Result, DatabaseError> { self.state.prepare_all(sql) } - pub fn ddl>(&mut self, sql: T) -> Result<(), DatabaseError> { + pub async fn ddl>(&mut self, sql: T) -> Result<(), DatabaseError> { let sql = sql.as_ref(); let statements = self .state @@ -191,45 +193,47 @@ impl Database { .with_sql_context(sql)); } } - self.run_mut_statements(sql, statements, |_| Ok(())) + self.run_mut_statements(sql, statements, ()).await } /// Executes SQL with catalog mutations allowed, lending the last statement's results /// to `consume`. On success, unread rows are drained before committing and publishing /// catalog updates. A callback or execution error rolls the current statement back. /// Earlier statements are completed and committed in order before the callback runs. - pub fn run_mut(&mut self, sql: impl AsRef, consume: C) -> Result + pub async fn run_mut( + &mut self, + sql: impl AsRef, + consume: C, + ) -> Result where - C: for<'a> FnOnce( - &mut TransactionIter<'a, 'a, S::TransactionType<'a>, PlanArena<'a>>, - ) -> Result, + C: ConsumeMut, { let sql = sql.as_ref(); let statements = self .state .prepare_all(sql) .map_err(|err| err.with_sql_context(sql))?; - self.run_mut_statements(sql, statements, consume) + self.run_mut_statements(sql, statements, consume).await } - fn run_mut_statements( + async fn run_mut_statements( &mut self, sql: &str, mut statements: Vec, consume: C, - ) -> Result + ) -> Result where - C: for<'a> FnOnce( - &mut TransactionIter<'a, 'a, S::TransactionType<'a>, PlanArena<'a>>, - ) -> Result, + C: ConsumeMut, { let last = statements .pop() .ok_or_else(|| DatabaseError::EmptyStatement.with_sql_context(sql))?; for statement in statements { - self.execute_mut(sql, &[], |binder, arena| binder.bind(&statement, arena))?; + self.execute_mut(sql, &[], |binder, arena| binder.bind(&statement, arena)) + .await?; } self.execute_mut_with(sql, &[], |binder, arena| binder.bind(&last, arena), consume) + .await } /// Runs one or more SQL statements and returns an iterator for the final result set. @@ -248,7 +252,7 @@ impl Database { /// iter.schema(|schema| assert_eq!(schema.len(), 1)); /// iter.done().unwrap(); /// ``` - pub fn run>(&self, sql: T) -> Result, DatabaseError> { + pub async fn run>(&self, sql: T) -> Result, DatabaseError> { let sql = sql.as_ref(); let statements = self .state @@ -276,47 +280,48 @@ impl Database { Ok::<_, DatabaseError>(read_only && statement_is_read_only(stmt)?) }) .map_err(|err| err.with_sql_context(sql))?; - let transaction = Box::into_raw(Box::new( + let transaction = OwnedPtr::new( self.storage .transaction_with_isolation(self.transaction_isolation, read_only)?, - )); + ); let mut statements = statements.into_iter().peekable(); while let Some(statement) = statements.next() { - let (plan_arena, executor) = match (|| { - let tx = unsafe { &mut *transaction }; + let (plan_arena, executor) = match async { + let tx = unsafe { &mut *transaction.get() }; let (plan, arena) = self .state .build_plan(&[], tx, |binder, arena| binder.bind(&statement, arena))?; let executor = Executor::new(plan, &arena); - self.state.execute(tx, executor, arena) - })() { + self.state.execute(tx, executor, arena).await + } + .await + { Ok(result) => result, - Err(err) => { - unsafe { drop(Box::from_raw(transaction)) }; - return Err(err.with_sql_context(sql)); - } + Err(err) => return Err(err.with_sql_context(sql)), }; if statements.peek().is_some() { - if let Err(err) = TransactionIter::new(plan_arena, executor).done() { - unsafe { drop(Box::from_raw(transaction)) }; + if let Err(err) = TransactionIter::new(plan_arena, executor).done().await { return Err(err.with_sql_context(sql)); } } else { - let inner = Box::into_raw(Box::new(TransactionIter::new(plan_arena, executor))); - return Ok(DatabaseIter { transaction, inner }); + let inner = Some(OwnedPtr::new(TransactionIter::new(plan_arena, executor))); + return Ok(DatabaseIter { + transaction: Some(transaction), + inner, + }); } } - unsafe { drop(Box::from_raw(transaction)) }; Err(DatabaseError::EmptyStatement.with_sql_context(sql)) } } +#[maybe_async] impl<'txn, S: Storage> Transaction<'txn, S> { /// Runs SQL inside the current transaction and returns the final result iterator. - pub fn run<'a, T: AsRef>( + pub async fn run<'a, T: AsRef>( &'a mut self, sql: T, ) -> Result>, DatabaseError> { @@ -343,8 +348,10 @@ impl<'txn, S: Storage> Transaction<'txn, S> { .build_plan(&[], tx, |binder, arena| binder.bind(&statement, arena)) .map(Executor::from_plan) }) + .await .map_err(|err| err.with_sql_context(sql))? .done() + .await .map_err(|err| err.with_sql_context(sql))?; } @@ -353,6 +360,7 @@ impl<'txn, S: Storage> Transaction<'txn, S> { .build_plan(&[], tx, |binder, arena| binder.bind(&last_statement, arena)) .map(Executor::from_plan) }) + .await .map_err(|err| err.with_sql_context(sql)) } } @@ -3379,6 +3387,7 @@ impl<'a, 'parent, T: StorageTransaction, A: AsRef<[(usize, LogicalType)]>> // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::binder::test::build_t1_table; @@ -3404,8 +3413,8 @@ mod tests { } #[test] - fn prepare_rejects_non_positional_parameters() -> Result<(), DatabaseError> { - let db = DataBaseBuilder::path(".").build_in_memory()?; + async fn prepare_rejects_non_positional_parameters() -> Result<(), DatabaseError> { + let db = DataBaseBuilder::path(".").build_in_memory().await?; for placeholder in ["$0", "$01", "$name"] { assert!(matches!( db.program_sql( @@ -3419,8 +3428,8 @@ mod tests { } #[test] - fn test_prepare_and_command_type_classification() -> Result<(), DatabaseError> { - let database = DataBaseBuilder::path(".").build_in_memory()?; + async fn test_prepare_and_command_type_classification() -> Result<(), DatabaseError> { + let database = DataBaseBuilder::path(".").build_in_memory().await?; assert!(matches!( database.prepare_all(""), Err(DatabaseError::EmptyStatement) @@ -3482,8 +3491,8 @@ mod tests { } #[test] - fn test_database_entrypoints_reject_catalog_mutation() -> Result<(), DatabaseError> { - let mut database = DataBaseBuilder::path(".").build_in_memory()?; + async fn test_database_entrypoints_reject_catalog_mutation() -> Result<(), DatabaseError> { + let mut database = DataBaseBuilder::path(".").build_in_memory().await?; let params = &[] as &[(usize, LogicalType)]; assert_unsupported( @@ -3491,17 +3500,20 @@ mod tests { "DDL and ANALYZE", ); assert_unsupported( - expect_err(database.run("create table t (id int primary key)")), + expect_err(database.run("create table t (id int primary key)").await), "DDL and ANALYZE", ); - assert_unsupported(expect_err(database.ddl("select 1")), "`Database::ddl`"); + assert_unsupported( + expect_err(database.ddl("select 1").await), + "`Database::ddl`", + ); let transaction = database.new_transaction(false)?; assert_unsupported( expect_err(transaction.program_sql("analyze table t", params)), "DDL and ANALYZE", ); - transaction.commit()?; + transaction.commit().await?; Ok(()) } @@ -3537,8 +3549,8 @@ mod tests { } #[test] - fn test_alter_table_parser_errors() -> Result<(), DatabaseError> { - let tables = build_t1_table()?; + async fn test_alter_table_parser_errors() -> Result<(), DatabaseError> { + let tables = build_t1_table().await?; assert_unsupported( tables @@ -3653,8 +3665,9 @@ mod tests { } #[test] - fn test_limit_parser_rejects_negative_and_non_constant_values() -> Result<(), DatabaseError> { - let tables = build_t1_table()?; + async fn test_limit_parser_rejects_negative_and_non_constant_values( + ) -> Result<(), DatabaseError> { + let tables = build_t1_table().await?; assert!(matches!( tables.plan("select * from t1 limit -1").unwrap_err(), @@ -3673,8 +3686,8 @@ mod tests { } #[test] - fn test_bind_non_recursive_ctes() -> Result<(), DatabaseError> { - let tables = build_t1_table()?; + async fn test_bind_non_recursive_ctes() -> Result<(), DatabaseError> { + let tables = build_t1_table().await?; let mut arena = PlanArena::new(&tables.table_arena); let mut plan = tables.plan_with_arena( "with first(a) as (select c1 from t1), \ @@ -3703,8 +3716,8 @@ mod tests { } #[test] - fn force_nest_loop_join_marks_join_operator() -> Result<(), DatabaseError> { - let tables = build_t1_table()?; + async fn force_nest_loop_join_marks_join_operator() -> Result<(), DatabaseError> { + let tables = build_t1_table().await?; let plan = tables.plan("select /*+ FORCE_NEST_LOOP_JOIN */ c1, c3 from t1 join t2 on c1 = c3")?; let join = plan @@ -3722,8 +3735,8 @@ mod tests { #[cfg(feature = "spill")] #[test] - fn optimizer_hints_can_be_combined() -> Result<(), DatabaseError> { - let tables = build_t1_table()?; + async fn optimizer_hints_can_be_combined() -> Result<(), DatabaseError> { + let tables = build_t1_table().await?; let plan = tables.plan( "select /*+ FORCE_AGG_SPILL, FORCE_NEST_LOOP_JOIN */ c1, count(c3) \ from t1 join t2 on c1 = c3 group by c1", diff --git a/src/binder/select.rs b/src/binder/select.rs index 9f4e23ce..65628320 100644 --- a/src/binder/select.rs +++ b/src/binder/select.rs @@ -2177,6 +2177,7 @@ impl<'a: 'b, 'b, T: StorageTransaction, A: AsRef<[(usize, LogicalType)]>> Binder // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::{InputPositionBinder, ProjectionOutputBinder}; use crate::binder::test::build_t1_table; @@ -2202,8 +2203,8 @@ mod tests { } #[test] - fn test_select_bind() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_select_bind() -> Result<(), DatabaseError> { + let table_states = build_t1_table().await?; let plan_1 = table_states.plan("select * from t1")?; println!("just_col:\n {plan_1:#?}"); @@ -2366,8 +2367,8 @@ mod tests { } #[test] - fn test_scalar_subquery_in_where_binds_as_init() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_scalar_subquery_in_where_binds_as_init() -> Result<(), DatabaseError> { + let table_states = build_t1_table().await?; let mut arena = PlanArena::new(&table_states.table_arena); let mut plan = table_states.plan_with_arena( "select * from t1 where c1 = (select max(c3) from t2)", @@ -2392,8 +2393,8 @@ mod tests { } #[test] - fn test_in_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_in_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { + let table_states = build_t1_table().await?; let plan = table_states.plan("select * from t1 where c1 in (select c3 from t2)")?; assert_quantified_mark_apply(&plan, MarkApplyQuantifier::Any, 1); @@ -2401,8 +2402,8 @@ mod tests { } #[test] - fn test_select_mark_alias_is_produced_before_where() -> Result<(), DatabaseError> { - let tables = build_t1_table()?; + async fn test_select_mark_alias_is_produced_before_where() -> Result<(), DatabaseError> { + let tables = build_t1_table().await?; let mut arena = PlanArena::new(&tables.table_arena); let mut plan = tables.plan_with_arena( "select c1,c2 in (select c3 from t2) as matched from t1 where matched", @@ -2449,8 +2450,8 @@ mod tests { } #[test] - fn test_any_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_any_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { + let table_states = build_t1_table().await?; let plan = table_states.plan("select * from t1 where c1 < any(select c3 from t2)")?; assert_quantified_mark_apply(&plan, MarkApplyQuantifier::Any, 1); @@ -2458,8 +2459,8 @@ mod tests { } #[test] - fn test_some_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_some_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { + let table_states = build_t1_table().await?; let plan = table_states.plan("select * from t1 where c1 = some(select c3 from t2)")?; assert_quantified_mark_apply(&plan, MarkApplyQuantifier::Any, 1); @@ -2467,8 +2468,8 @@ mod tests { } #[test] - fn test_all_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_all_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { + let table_states = build_t1_table().await?; let plan = table_states.plan("select * from t1 where c1 > all(select c3 from t2)")?; assert_quantified_mark_apply(&plan, MarkApplyQuantifier::All, 1); @@ -2476,8 +2477,9 @@ mod tests { } #[test] - fn test_correlated_in_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_correlated_in_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> + { + let table_states = build_t1_table().await?; let plan = table_states.plan("select * from t1 where c1 in (select c3 from t2 where c4 = c2)")?; assert_quantified_mark_apply(&plan, MarkApplyQuantifier::Any, 2); @@ -2486,8 +2488,9 @@ mod tests { } #[test] - fn test_correlated_any_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_correlated_any_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> + { + let table_states = build_t1_table().await?; let plan = table_states .plan("select * from t1 where c1 < any(select c3 from t2 where c4 = c2)")?; assert_quantified_mark_apply(&plan, MarkApplyQuantifier::Any, 2); @@ -2496,8 +2499,9 @@ mod tests { } #[test] - fn test_correlated_all_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_correlated_all_subquery_in_where_binds_as_mark_apply() -> Result<(), DatabaseError> + { + let table_states = build_t1_table().await?; let plan = table_states .plan("select * from t1 where c1 > all(select c3 from t2 where c4 = c2)")?; assert_quantified_mark_apply(&plan, MarkApplyQuantifier::All, 2); @@ -2506,8 +2510,8 @@ mod tests { } #[test] - fn test_multiple_scalar_subqueries_have_independent_slots() -> Result<(), DatabaseError> { - let table_states = build_t1_table()?; + async fn test_multiple_scalar_subqueries_have_independent_slots() -> Result<(), DatabaseError> { + let table_states = build_t1_table().await?; let mut arena = PlanArena::new(&table_states.table_arena); let plan = table_states.plan_with_arena( "select * from t1 where c1 <= (select 4) and c1 > (select 1)", diff --git a/src/catalog/column.rs b/src/catalog/column.rs index d3fa2417..ba834972 100644 --- a/src/catalog/column.rs +++ b/src/catalog/column.rs @@ -242,6 +242,7 @@ impl ColumnCatalog { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::{ColumnCatalog, ColumnDesc}; use crate::errors::DatabaseError; diff --git a/src/catalog/table.rs b/src/catalog/table.rs index 801aa1e1..8ff9edc1 100644 --- a/src/catalog/table.rs +++ b/src/catalog/table.rs @@ -405,6 +405,7 @@ impl TableMeta { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::ColumnDesc; diff --git a/src/db.rs b/src/db.rs index 3f3028fa..b82f3e20 100644 --- a/src/db.rs +++ b/src/db.rs @@ -18,7 +18,7 @@ pub use crate::binder::Statement; use crate::binder::{Binder, BinderContext}; use crate::errors::DatabaseError; pub use crate::execution::Executor; -use crate::execution::{DDLApply, ExecArena, ExecBuffer}; +use crate::execution::{DDLApply, ExecArena, ExecBuffer, TxnPtr}; use crate::expression::function::scala::ScalarFunctionImpl; use crate::expression::function::table::{ ArcTableFunctionImpl, TableFunctionCatalog, TableFunctionImpl, @@ -34,6 +34,7 @@ use crate::function::lower::Lower; use crate::function::numbers::Numbers; use crate::function::octet_length::OctetLength; use crate::function::upper::Upper; +use crate::maybe_async::MaybeSend; use crate::optimizer::core::hll; use crate::optimizer::core::statistics_meta::StatisticMetaLoader; use crate::optimizer::heuristic::batch::HepBatchStrategy; @@ -46,6 +47,7 @@ use crate::planner::operator::Operator; use crate::planner::{ExecMetaArena, LogicalPlan, MetaArena, PlanArena, TableArenaCell}; #[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))] use crate::storage::lmdb::{LmdbConfig, LmdbStorage}; +#[cfg(not(feature = "async"))] use crate::storage::memory::MemoryStorage; #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] use crate::storage::rocksdb::{OptimisticRocksStorage, RocksStorage, StorageConfig}; @@ -69,6 +71,37 @@ use std::path::PathBuf; use std::sync::Arc; /// Metadata arena, and the executor with the arena it runs in. +pub(crate) struct OwnedPtr(*mut T); + +// SAFETY: `OwnedPtr` owns its value like `Box`. +unsafe impl Send for OwnedPtr {} +// SAFETY: as for `Send`. +unsafe impl Sync for OwnedPtr {} + +impl OwnedPtr { + pub(crate) fn new(value: T) -> Self { + Self(Box::into_raw(Box::new(value))) + } + + pub(crate) fn get(&self) -> *mut T { + self.0 + } + + /// # Safety + /// No reference obtained through [`get`](Self::get) may still be in use. + pub(crate) unsafe fn into_inner(self) -> T { + let this = std::mem::ManuallyDrop::new(self); + *Box::from_raw(this.0) + } +} + +impl Drop for OwnedPtr { + fn drop(&mut self) { + // SAFETY: `self.0` comes from `Box::into_raw` and is freed only here or in `into_inner`. + drop(unsafe { Box::from_raw(self.0) }); + } +} + pub(crate) type Execution<'a, 'p, A, T> = (ExecMetaArena, (Executor<'p>, ExecArena<'p, 'a, T>)); pub(crate) type ScalaFunctions = HashMap>; @@ -81,7 +114,8 @@ pub enum CatalogKind { TableFunction(Arc), } -pub(crate) trait BindSource<'a> { +#[maybe_async] +pub(crate) trait BindSource { type Iter: ResultIter; type Transaction: StorageTransaction; @@ -89,12 +123,13 @@ pub(crate) trait BindSource<'a> { /// `read_only` lets a fresh transaction use the storage's read path; executors /// that already hold a transaction ignore it. - fn execute(self, read_only: bool, build: F) -> Result + async fn execute(self, read_only: bool, build: F) -> Result where - F: FnOnce( - &'a State, - &Self::Transaction, - ) -> Result<(Executor<'a>, PlanArena<'a>), DatabaseError>; + F: for<'s> FnOnce( + &'s State, + &Self::Transaction, + ) -> Result<(Executor<'s>, PlanArena<'s>), DatabaseError> + + MaybeSend; #[cfg(feature = "orm")] fn explain(self, params: A, build: F) -> Result @@ -319,6 +354,7 @@ impl DataBaseBuilder { /// Builds an in-memory database. /// /// This is convenient for tests, examples and temporary workloads. + #[cfg(not(feature = "async"))] pub fn build_in_memory(self) -> Result, DatabaseError> { let storage = MemoryStorage::new(); @@ -614,29 +650,38 @@ fn statement_stamp( } } -fn start_execution<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p>( +#[maybe_async] +async fn start_execution<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p>( table_cache: &'p TableCache, transaction: &'a mut T, mut executor: Executor<'p>, plan_arena: A, -) -> Result, DatabaseError> { +) -> Result, DatabaseError> +where + 'p: 'a, +{ let mut plan_arena = ExecMetaArena::new(plan_arena); - let arena = start_execution_inner(table_cache, transaction, &mut executor, &mut plan_arena)?; + let arena = + start_execution_inner(table_cache, transaction, &mut executor, &mut plan_arena).await?; Ok((plan_arena, (executor, arena))) } -fn start_execution_inner<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p>( +#[maybe_async] +async fn start_execution_inner<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p>( table_cache: &'p TableCache, transaction: &'a mut T, executor: &mut Executor<'p>, plan_arena: &mut ExecMetaArena, -) -> Result, DatabaseError> { +) -> Result, DatabaseError> +where + 'p: 'a, +{ let stamp = statement_stamp(transaction, executor.plan())?; let iters = executor.buffer.iter_slots(executor.iters()); let mut arena = ExecArena::new(table_cache, transaction).with_iters(iters); executor.buffer.set_statement_stamp(stamp); - executor.start(&mut arena, plan_arena)?; + executor.start(&mut arena, plan_arena).await?; Ok(arena) } @@ -781,8 +826,11 @@ impl State { Ok((plan, arena)) } +} - pub(crate) fn execute<'a, 'p, 'txn, A: MetaArena + 'p>( +#[maybe_async] +impl State { + pub(crate) async fn execute<'a, 'p, 'txn, A: MetaArena + 'p>( &'p self, transaction: &'a mut S::TransactionType<'txn>, executor: Executor<'p>, @@ -793,10 +841,10 @@ impl State { 'txn: 'p, 'p: 'a, { - start_execution(self.table_cache(), transaction, executor, plan_arena) + start_execution(self.table_cache(), transaction, executor, plan_arena).await } - pub(crate) fn execute_mut<'a, 'txn, A, F>( + pub(crate) async fn execute_mut<'a, 'txn, A, F>( &'a mut self, transaction: &'a mut S::TransactionType<'txn>, params: A, @@ -856,7 +904,7 @@ impl State { plan.populate_output_schema_recursive(&mut plan_arena); let executor = Executor::new(plan, &plan_arena); - start_execution(table_cache, transaction, executor, plan_arena) + start_execution(table_cache, transaction, executor, plan_arena).await } } @@ -928,8 +976,9 @@ impl DDLApply { } } +#[maybe_async] impl Database { - pub(crate) fn execute_mut( + pub(crate) async fn execute_mut( &mut self, context: &str, params: A, @@ -942,76 +991,74 @@ impl Database { &mut PlanArena<'a>, ) -> Result, { - self.execute_mut_with(context, params, build, |_| Ok(())) + self.execute_mut_with(context, params, build, ()).await } - pub(crate) fn execute_mut_with( + pub(crate) async fn execute_mut_with( &mut self, context: &str, params: A, build: F, consume: C, - ) -> Result + ) -> Result where A: AsRef<[(usize, LogicalType)]>, F: for<'a, 'txn, 'bind> FnOnce( &mut Binder<'bind, '_, S::TransactionType<'txn>, A>, &mut PlanArena<'a>, ) -> Result, - C: for<'a> FnOnce( - &mut TransactionIter<'a, 'a, S::TransactionType<'a>, PlanArena<'a>>, - ) -> Result, + C: ConsumeMut, { - let transaction = Box::into_raw(Box::new( + let transaction = OwnedPtr::new( self.storage .transaction_with_isolation(self.transaction_isolation, false)?, - )); - let state = std::ptr::from_mut(&mut self.state); - let (plan_arena, executor) = - match unsafe { (&mut *state).execute_mut(&mut *transaction, params, build) } { - Ok(result) => result, - Err(err) => { - unsafe { drop(Box::from_raw(transaction)) }; - return Err(err.with_sql_context(context)); - } - }; - let (result, mut plan_arena, apply) = match (|| { + ); + let state = TxnPtr::new(std::ptr::from_mut(&mut self.state)); + let (plan_arena, executor) = match unsafe { + (&mut *state.get()) + .execute_mut(&mut *transaction.get(), params, build) + .await + } { + Ok(result) => result, + Err(err) => return Err(err.with_sql_context(context)), + }; + let (result, mut plan_arena, apply) = match async { let mut iter = TransactionIter::new(plan_arena, executor); - let result = consume(&mut iter)?; - let (arena, apply) = iter.done_with_ddl_apply()?; + let result = consume.consume(&mut iter).await?; + let (arena, apply) = iter.done_with_ddl_apply().await?; Ok::<_, DatabaseError>((result, arena, apply)) - })() { + } + .await + { Ok(result) => result, - Err(err) => { - unsafe { drop(Box::from_raw(transaction)) }; - return Err(err.with_sql_context(context)); - } + Err(err) => return Err(err.with_sql_context(context)), }; - if let Err(err) = unsafe { Box::from_raw(transaction).commit() } { + if let Err(err) = unsafe { transaction.into_inner() }.commit().await { return Err(err.with_sql_context(context)); } let mut catalog_changed = false; for apply in apply { - catalog_changed |= unsafe { apply.apply_to(&mut *state, &mut plan_arena) } + catalog_changed |= unsafe { apply.apply_to(&mut *state.get(), &mut plan_arena) } .map_err(|err| err.with_sql_context(context))?; } if catalog_changed { - unsafe { (&mut *state).recycle_table_arena() }?; + unsafe { (&mut *state.get()).recycle_table_arena() }?; } Ok(result) } - pub fn analyze(&mut self, table_name: impl AsRef) -> Result<(), DatabaseError> { + pub async fn analyze(&mut self, table_name: impl AsRef) -> Result<(), DatabaseError> { let context = "ANALYZE"; let table_name = table_name.as_ref(); self.execute_mut(context, &[], move |binder, arena| { binder.bind_analyze(table_name, arena) }) + .await } - pub fn load(&mut self, kind: CatalogKind) -> Result<(), DatabaseError> { + pub async fn load(&mut self, kind: CatalogKind) -> Result<(), DatabaseError> { match kind { CatalogKind::ScalarFunction(function) => { self.state.load_scalar_function(function); @@ -1029,18 +1076,22 @@ impl Database { )?; let mut table_codec = TableCodec::default(); let table = transaction - .load_table(&mut table_codec, self.state.table_arena.borrow_mut(), &name)? + .load_table(&mut table_codec, self.state.table_arena.borrow_mut(), &name) + .await? .ok_or(DatabaseError::TableNotFound)?; let mut indexes = Vec::new(); let mut columns = Vec::new(); for index in table.indexes() { let index_id = self.state.table_arena.borrow().index(*index).id; - if let Some(meta) = transaction.index_statistics( - &mut table_codec, - table.id(), - index_id, - self.state.table_arena.borrow_mut(), - )? { + if let Some(meta) = transaction + .index_statistics( + &mut table_codec, + table.id(), + index_id, + self.state.table_arena.borrow_mut(), + ) + .await? + { indexes.push((index_id, meta)); } } @@ -1052,12 +1103,15 @@ impl Database { .id() .ok_or_else(|| DatabaseError::invalid_column(column.name()))? }; - if let Some(meta) = transaction.column_statistics( - &mut table_codec, - table.id(), - column_id, - self.state.table_arena.borrow_mut(), - )? { + if let Some(meta) = transaction + .column_statistics( + &mut table_codec, + table.id(), + column_id, + self.state.table_arena.borrow_mut(), + ) + .await? + { columns.push((column_id, meta)); } } @@ -1088,7 +1142,8 @@ impl Database { &self.state.scala_functions, &self.state.table_functions, name.clone(), - )? + ) + .await? .ok_or(DatabaseError::ViewNotFound)?; self.state .table_arena @@ -1135,37 +1190,55 @@ impl Database { } } -impl<'a, S: Storage> BindSource<'a> for &'a Database { +#[maybe_async] +pub trait ConsumeMut { + type Output; + + async fn consume<'a>( + self, + iter: &mut TransactionIter<'a, 'a, S::TransactionType<'a>, PlanArena<'a>>, + ) -> Result; +} + +#[maybe_async] +impl ConsumeMut for () { + type Output = (); + + async fn consume<'a>( + self, + _: &mut TransactionIter<'a, 'a, S::TransactionType<'a>, PlanArena<'a>>, + ) -> Result<(), DatabaseError> { + Ok(()) + } +} + +#[maybe_async] +impl<'a, S: Storage> BindSource for &'a Database { type Iter = DatabaseIter<'a, S>; type Transaction = S::TransactionType<'a>; type Storage = S; - fn execute(self, read_only: bool, build: F) -> Result + async fn execute(self, read_only: bool, build: F) -> Result where - F: FnOnce( - &'a State, - &Self::Transaction, - ) -> Result<(Executor<'a>, PlanArena<'a>), DatabaseError>, + F: for<'s> FnOnce( + &'s State, + &Self::Transaction, + ) -> Result<(Executor<'s>, PlanArena<'s>), DatabaseError> + + MaybeSend, { - let transaction = Box::into_raw(Box::new( + let transaction = OwnedPtr::new( self.storage .transaction_with_isolation(self.transaction_isolation, read_only)?, - )); - let result = (|| { - let transaction = unsafe { &mut *transaction }; - let (executor, arena) = build(&self.state, transaction)?; - self.state.execute(transaction, executor, arena) - })(); - let (arena, executor) = match result { - Ok(result) => result, - Err(error) => { - unsafe { drop(Box::from_raw(transaction)) }; - return Err(error); - } - }; - let inner = Box::into_raw(Box::new(TransactionIter::new(arena, executor))); - Ok(DatabaseIter { transaction, inner }) + ); + // SAFETY: the transaction stays boxed, and so in place, until the iterator is gone. + let tx = unsafe { &mut *transaction.get() }; + let (executor, arena) = build(&self.state, tx)?; + let (arena, executor) = self.state.execute(tx, executor, arena).await?; + Ok(DatabaseIter { + inner: Some(OwnedPtr::new(TransactionIter::new(arena, executor))), + transaction: Some(transaction), + }) } #[cfg(feature = "orm")] @@ -1314,12 +1387,13 @@ impl<'txn> TxnValue<'txn> { /// A result iterator that borrows its transaction for `'txn`, such as [`TransactionIter`], so its /// rows lend values for as long. +#[maybe_async] pub trait TxnResultIter<'txn>: ResultIter { /// Like [`ResultIter::next_tuple`], but the row lends values for `'txn`. The iterators' own /// `next_tuple` does the same; this is for code generic over them. - fn next_txn_tuple( + async fn next_txn_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'txn>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'txn>) -> R + MaybeSend, ) -> Result, DatabaseError>; #[cfg(feature = "orm")] @@ -1333,18 +1407,19 @@ pub trait TxnResultIter<'txn>: ResultIter { } /// Common interface for result iterators returned by database execution APIs. +#[maybe_async] pub trait ResultIter { /// Borrows the output schema for the current result set. fn schema(&self, f: impl FnOnce(&SchemaView<'_, '_>) -> R) -> R; /// Advances to the next row and runs `f` while the executor row slot is borrowed. - fn next_tuple( + async fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R + MaybeSend, ) -> Result, DatabaseError>; /// Finishes consuming the iterator and flushes any remaining work. - fn done(self) -> Result<(), DatabaseError>; + async fn done(self) -> Result<(), DatabaseError>; #[cfg(feature = "orm")] /// Converts this iterator into a typed ORM iterator. @@ -1361,6 +1436,21 @@ pub trait ResultIter { } } +#[maybe_async] +#[cfg(feature = "orm")] +#[sync_only] +pub trait AsyncIter: Iterator {} + +#[maybe_async] +#[cfg(feature = "orm")] +#[async_only] +pub trait AsyncIter { + type Item; + + /// Reads the next item, or `None` once the rows run out. + async fn next(&mut self) -> Option; +} + #[cfg(feature = "orm")] /// Typed adapter over a [`ResultIter`] that yields ORM models instead of raw tuples. pub struct OrmIter { @@ -1368,6 +1458,7 @@ pub struct OrmIter { _marker: PhantomData, } +#[maybe_async] #[cfg(feature = "orm")] impl OrmIter where @@ -1387,12 +1478,35 @@ where } /// Finishes the underlying raw iterator. - pub fn done(self) -> Result<(), DatabaseError> { - self.inner.done() + pub async fn done(self) -> Result<(), DatabaseError> { + self.inner.done().await + } +} + +#[maybe_async] +#[cfg(feature = "orm")] +impl AsyncIter for OrmIter +where + I: ResultIter + MaybeSend, + T: FromQueryRow + MaybeSend, +{ + #[async_only] + type Item = Result; + + /// Reads the next model. + #[async_only] + async fn next(&mut self) -> Option { + self.inner + .next_tuple(|schema, tuple| T::from_query_row(schema, tuple)) + .await + .transpose() + .map(|row| row.and_then(std::convert::identity)) } } +#[maybe_async] #[cfg(feature = "orm")] +#[sync_only] impl Iterator for OrmIter where I: ResultIter, @@ -1414,6 +1528,7 @@ pub struct TxnOrmIter<'txn, I, T> { _marker: PhantomData<(&'txn (), T)>, } +#[maybe_async] #[cfg(feature = "orm")] impl<'txn, I, T> TxnOrmIter<'txn, I, T> where @@ -1427,12 +1542,39 @@ where } } - pub fn done(self) -> Result<(), DatabaseError> { - self.inner.done() + pub async fn done(self) -> Result<(), DatabaseError> { + self.inner.done().await } } +#[maybe_async] #[cfg(feature = "orm")] +impl<'txn, I, T> AsyncIter for TxnOrmIter<'txn, I, T> +where + I: TxnResultIter<'txn> + MaybeSend, + T: FromTxnRow<'txn> + MaybeSend, +{ + #[async_only] + type Item = Result; + + /// Reads the next model. + #[async_only] + async fn next(&mut self) -> Option { + // Boxed as `dyn Future + Send` here, where `I: TxnResultIter<'txn>` holds: inside the + // future of `next` the compiler loses that bound when it checks `Send`. + let row: std::pin::Pin + Send + '_>> = Box::pin( + self.inner + .next_txn_tuple(|schema, row| T::from_txn_row(schema, row)), + ); + row.await + .transpose() + .map(|row| row.and_then(std::convert::identity)) + } +} + +#[maybe_async] +#[cfg(feature = "orm")] +#[sync_only] impl<'txn, I, T> Iterator for TxnOrmIter<'txn, I, T> where I: TxnResultIter<'txn>, @@ -1450,61 +1592,67 @@ where /// Raw result iterator returned by database execution APIs. pub struct DatabaseIter<'a, S: Storage + 'a> { - pub(crate) transaction: *mut S::TransactionType<'a>, - pub(crate) inner: *mut TransactionIter<'a, 'a, S::TransactionType<'a>>, + // Dropped before `transaction`, which it borrows; `done` takes both out. + pub(crate) inner: Option>>>, + pub(crate) transaction: Option>>, } -impl Drop for DatabaseIter<'_, S> { - fn drop(&mut self) { - if !self.inner.is_null() { - unsafe { drop(Box::from_raw(self.inner)) } - } - if !self.transaction.is_null() { - unsafe { drop(Box::from_raw(self.transaction)) } - } +#[maybe_async] +impl<'a, S: Storage> DatabaseIter<'a, S> { + fn inner(&self) -> &TransactionIter<'a, 'a, S::TransactionType<'a>> { + let inner = self.inner.as_ref().expect("iterator still running"); + // SAFETY: `inner` is owned by `self`, and shared here as `self` is. + unsafe { &*inner.get() } + } + + fn inner_mut(&mut self) -> &mut TransactionIter<'a, 'a, S::TransactionType<'a>> { + let inner = self.inner.as_ref().expect("iterator still running"); + // SAFETY: `inner` is owned by `self`, which is borrowed mutably. + unsafe { &mut *inner.get() } } -} -impl<'a, S: Storage> DatabaseIter<'a, S> { #[inline] pub fn schema(&self, f: impl FnOnce(&SchemaView<'_, '_>) -> R) -> R { - unsafe { (*self.inner).schema(f) } + self.inner().schema(f) } #[inline] - pub fn next_tuple( + pub async fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R + MaybeSend, ) -> Result, DatabaseError> { - unsafe { (*self.inner).next_tuple(f) } + self.inner_mut().next_tuple(f).await } #[inline] - pub fn done(mut self) -> Result<(), DatabaseError> { - unsafe { - Box::from_raw(mem::replace(&mut self.inner, std::ptr::null_mut())).done()?; + pub async fn done(mut self) -> Result<(), DatabaseError> { + if let Some(inner) = self.inner.take() { + // SAFETY: taken out of `self`, so this is the only owner left. + unsafe { inner.into_inner() }.done().await?; } - unsafe { - Box::from_raw(mem::replace(&mut self.transaction, std::ptr::null_mut())).commit()?; + if let Some(transaction) = self.transaction.take() { + // SAFETY: the iterator that borrowed the transaction is gone. + unsafe { transaction.into_inner() }.commit().await?; } Ok(()) } } +#[maybe_async] impl ResultIter for DatabaseIter<'_, S> { fn schema(&self, f: impl FnOnce(&SchemaView<'_, '_>) -> R) -> R { DatabaseIter::schema(self, f) } - fn next_tuple( + async fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R + MaybeSend, ) -> Result, DatabaseError> { - DatabaseIter::next_tuple(self, f) + DatabaseIter::next_tuple(self, f).await } - fn done(self) -> Result<(), DatabaseError> { - DatabaseIter::done(self) + async fn done(self) -> Result<(), DatabaseError> { + DatabaseIter::done(self).await } } @@ -1515,27 +1663,30 @@ pub struct Transaction<'a, S: Storage + 'a> { read_only: bool, } +#[maybe_async] impl<'txn, S: Storage> Transaction<'txn, S> { /// Commits the current transaction. - pub fn commit(self) -> Result<(), DatabaseError> { - self.inner.commit()?; + pub async fn commit(self) -> Result<(), DatabaseError> { + self.inner.commit().await?; Ok(()) } } -impl<'a, 'txn, S: Storage> BindSource<'a> for &'a mut Transaction<'txn, S> { +#[maybe_async] +impl<'a, 'txn, S: Storage> BindSource for &'a mut Transaction<'txn, S> { type Iter = TransactionIter<'a, 'a, S::TransactionType<'txn>>; type Transaction = S::TransactionType<'txn>; type Storage = S; - fn execute(self, _read_only: bool, build: F) -> Result + async fn execute(self, _read_only: bool, build: F) -> Result where - F: FnOnce( - &'a State, - &Self::Transaction, - ) -> Result<(Executor<'a>, PlanArena<'a>), DatabaseError>, + F: for<'s> FnOnce( + &'s State, + &Self::Transaction, + ) -> Result<(Executor<'s>, PlanArena<'s>), DatabaseError> + + MaybeSend, { let (executor, arena) = build(self.state, &self.inner)?; if self.read_only && !executor.plan().operator.command_type().is_read_only() { @@ -1543,7 +1694,7 @@ impl<'a, 'txn, S: Storage> BindSource<'a> for &'a mut Transaction<'txn, S> { "cannot write in a read-only transaction".into(), )); } - let (arena, executor) = self.state.execute(&mut self.inner, executor, arena)?; + let (arena, executor) = self.state.execute(&mut self.inner, executor, arena).await?; Ok(TransactionIter::new(arena, executor)) } @@ -1577,13 +1728,14 @@ impl TransactionGuard<'_, '_, T> { } } +#[maybe_async] impl<'p, T: StorageTransaction + 'p> TransactionGuard<'_, 'p, T> { /// Pulls the next row of `executor`, finishing the execution once the rows run out /// or the execution fails. /// /// The row lends values for `'txn`, which the caller must not let outlive the transaction. #[inline] - fn next_tuple<'txn, A: MetaArena + 'p, R>( + async fn next_tuple<'txn, A: MetaArena + 'p, R>( &mut self, executor: &mut Executor<'p>, plan_arena: &mut ExecMetaArena, @@ -1592,7 +1744,7 @@ impl<'p, T: StorageTransaction + 'p> TransactionGuard<'_, 'p, T> { if !self.active { return Ok(None); } - match executor.next_tuple(&mut self.arena, plan_arena) { + match executor.next_tuple(&mut self.arena, plan_arena).await { Ok(true) => { let schema = SchemaView::new(&executor.schema, plan_arena); Ok(Some(f( @@ -1629,6 +1781,7 @@ pub struct TransactionIter<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p plan_arena: ExecMetaArena, } +#[maybe_async] impl<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p> TransactionIter<'a, 'p, T, A> { pub(crate) fn new( plan_arena: ExecMetaArena, @@ -1652,24 +1805,28 @@ impl<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p> TransactionIter<'a, /// Advances to the next row and runs `f` on it. The iterator borrows its transaction for /// `'a`, so the row lends values for as long (see [`ResultRow::take_txn_value`]). #[inline] - pub fn next_tuple( + pub async fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R + MaybeSend, ) -> Result, DatabaseError> { self.guard .next_tuple(&mut self.executor, &mut self.plan_arena, f) + .await } #[inline] - pub fn done(mut self) -> Result<(), DatabaseError> { - while self.next_tuple(|_, _| ())?.is_some() {} + pub async fn done(mut self) -> Result<(), DatabaseError> { + while self.next_tuple(|_, _| ()).await?.is_some() {} Ok(()) } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> TransactionIter<'a, 'a, T, PlanArena<'a>> { - fn done_with_ddl_apply(mut self) -> Result<(PlanArena<'a>, Vec), DatabaseError> { - while self.next_tuple(|_, _| ())?.is_some() {} + async fn done_with_ddl_apply( + mut self, + ) -> Result<(PlanArena<'a>, Vec), DatabaseError> { + while self.next_tuple(|_, _| ()).await?.is_some() {} let Self { mut guard, plan_arena, @@ -1681,6 +1838,7 @@ impl<'a, T: StorageTransaction + 'a> TransactionIter<'a, 'a, T, PlanArena<'a>> { } } +#[maybe_async] impl<'p, T: StorageTransaction + 'p, A: MetaArena + 'p> ResultIter for TransactionIter<'_, 'p, T, A> { @@ -1688,31 +1846,33 @@ impl<'p, T: StorageTransaction + 'p, A: MetaArena + 'p> ResultIter TransactionIter::schema(self, f) } - fn next_tuple( + async fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R + MaybeSend, ) -> Result, DatabaseError> { - TransactionIter::next_tuple(self, f) + TransactionIter::next_tuple(self, f).await } - fn done(self) -> Result<(), DatabaseError> { - TransactionIter::done(self) + async fn done(self) -> Result<(), DatabaseError> { + TransactionIter::done(self).await } } +#[maybe_async] impl<'a, 'p, T: StorageTransaction + 'p, A: MetaArena + 'p> TxnResultIter<'a> for TransactionIter<'a, 'p, T, A> { - fn next_txn_tuple( + async fn next_txn_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R + MaybeSend, ) -> Result, DatabaseError> { - TransactionIter::next_tuple(self, f) + TransactionIter::next_tuple(self, f).await } } // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] pub(crate) mod test { use crate::binder::{Binder, BinderContext}; use crate::catalog::{ColumnCatalog, ColumnDesc}; @@ -1727,15 +1887,17 @@ pub(crate) mod test { use crate::planner::operator::values::ValuesOperator; use crate::planner::operator::Operator; use crate::planner::{Childrens, ExprRef, LogicalPlan, PlanArena, TableArenaCell}; - use crate::storage::memory::MemoryStorage; + #[cfg(feature = "rocksdb")] + use crate::storage::TransactionIsolationLevel; use crate::storage::{ table_codec::TableCodec, NameMapping, StatisticsMetaCache, Storage, StorageTransaction, - TableCache, TransactionIsolationLevel, + TableCache, }; use crate::types::tuple::{Tuple, TupleImpl}; use crate::types::value::DataValue; use crate::types::LogicalType; use chrono::{Datelike, Local}; + #[cfg(feature = "rocksdb")] use std::io::ErrorKind; #[cfg(feature = "unsafe_txdb_checkpoint")] use std::sync::atomic::AtomicUsize; @@ -1788,7 +1950,8 @@ pub(crate) mod test { } #[test] - fn late_project_collapse_preserves_oriented_columns_and_aliases() -> Result<(), DatabaseError> { + async fn late_project_collapse_preserves_oriented_columns_and_aliases( + ) -> Result<(), DatabaseError> { for alias in [false, true] { let tables = TableArenaCell::default(); let mut arena = PlanArena::new(&tables); @@ -1835,7 +1998,7 @@ pub(crate) mod test { let Childrens::Only(input) = plan.childrens.as_ref() else { unreachable!() }; - let storage = MemoryStorage::new(); + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = TableCache::default(); let context = &table_cache; @@ -1843,7 +2006,7 @@ pub(crate) mod test { execute_input::<_, Projection>((project, input), context, arena, &transaction); for value in [1, 2, 3, 4] { assert_eq!( - executor.next_tuple()?.cloned().unwrap(), + executor.next_tuple().await?.cloned().unwrap(), vec![ DataValue::Int32(value), DataValue::Int32(value), @@ -1851,13 +2014,13 @@ pub(crate) mod test { ] ); } - assert!(executor.next_tuple()?.is_none()); + assert!(executor.next_tuple().await?.is_none()); } Ok(()) } #[test] - fn late_project_collapse_inlines_computed_output() -> Result<(), DatabaseError> { + async fn late_project_collapse_inlines_computed_output() -> Result<(), DatabaseError> { let tables = TableArenaCell::default(); let mut arena = PlanArena::new(&tables); let (left, large) = literal_input("large", &[1, 2, 3, 4], 0, &mut arena)?; @@ -1896,7 +2059,7 @@ pub(crate) mod test { let Childrens::Only(input) = plan.childrens.as_ref() else { unreachable!() }; - let storage = MemoryStorage::new(); + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = TableCache::default(); let context = &table_cache; @@ -1904,11 +2067,11 @@ pub(crate) mod test { execute_input::<_, Projection>((project, input), context, arena, &transaction); for value in [11, 12, 13, 14] { assert_eq!( - executor.next_tuple()?.cloned().unwrap(), + executor.next_tuple().await?.cloned().unwrap(), vec![DataValue::Int32(value), DataValue::Int32(99)] ); } - assert!(executor.next_tuple()?.is_none()); + assert!(executor.next_tuple().await?.is_none()); Ok(()) } @@ -1916,6 +2079,8 @@ pub(crate) mod test { #[test] fn database_handles_are_send_sync() { + #[cfg(feature = "async")] + assert_send_sync::>(); #[cfg(feature = "rocksdb")] assert_send_sync::>(); #[cfg(feature = "lmdb")] @@ -1923,25 +2088,28 @@ pub(crate) mod test { } #[test] - fn scalar_query_view_references_survive_catalog_reload() -> Result<(), DatabaseError> { - let storage = MemoryStorage::new(); + async fn scalar_query_view_references_survive_catalog_reload() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; { let mut db = DataBaseBuilder::path(".").build_with_storage(storage.clone())?; - db.ddl("create view scalar_a as select (select 11) as v")?; - db.ddl("create view scalar_b as select (select 22) as v")?; - db.ddl("create view scalar_nested as select (select (select 33)) as v")?; + db.ddl("create view scalar_a as select (select 11) as v") + .await?; + db.ddl("create view scalar_b as select (select 22) as v") + .await?; + db.ddl("create view scalar_nested as select (select (select 33)) as v") + .await?; } let mut db = DataBaseBuilder::path(".").build_with_storage(storage)?; // Loading in a different order must preserve the saved initialization references. for name in ["scalar_b", "scalar_a", "scalar_nested"] { - db.load(CatalogKind::View(name.into()))?; + db.load(CatalogKind::View(name.into())).await?; } let sql = "select a.v, b.v, c.v, (select 44) from scalar_a a \ cross join scalar_b b cross join scalar_nested c"; - let mut iter = db.run(sql)?; + let mut iter = db.run(sql).await?; assert_eq!( - iter.next_tuple(|_, row| row.clone())?.unwrap(), + iter.next_tuple(|_, row| row.clone()).await?.unwrap(), vec![ DataValue::Int32(11), DataValue::Int32(22), @@ -1949,25 +2117,28 @@ pub(crate) mod test { DataValue::Int32(44), ] ); - assert!(iter.next_tuple(|_, _| ())?.is_none()); - iter.done()?; + assert!(iter.next_tuple(|_, _| ()).await?.is_none()); + iter.done().await?; // New definitions must not reuse references registered by the loaded views. - db.ddl("create view scalar_new as select (select 55) as v")?; - let mut iter = db.run("select a.v, b.v from scalar_a a cross join scalar_new b")?; + db.ddl("create view scalar_new as select (select 55) as v") + .await?; + let mut iter = db + .run("select a.v, b.v from scalar_a a cross join scalar_new b") + .await?; assert_eq!( - iter.next_tuple(|_, row| row.clone())?.unwrap(), + iter.next_tuple(|_, row| row.clone()).await?.unwrap(), vec![DataValue::Int32(11), DataValue::Int32(55)] ); - iter.done()?; + iter.done().await?; Ok(()) } - pub(crate) fn build_table( + pub(crate) async fn build_table( names: &mut NameMapping, table_cache: &mut TableCache, transaction: &mut T, - plan_arena: &mut PlanArena, + plan_arena: &mut PlanArena<'_>, ) -> Result<(), DatabaseError> { let columns = vec![ ColumnCatalog::new( @@ -1987,13 +2158,16 @@ pub(crate) mod test { ), ]; let mut table_codec = TableCodec::default(); - if let Some(table) = transaction.create_table( - &mut table_codec, - plan_arena, - "t1".to_string(), - columns, - false, - )? { + if let Some(table) = transaction + .create_table( + &mut table_codec, + plan_arena, + "t1".to_string(), + columns, + false, + ) + .await? + { let table = table.transplant_to_table_arena(plan_arena)?; names.insert_table(table_cache, table); } @@ -2001,26 +2175,31 @@ pub(crate) mod test { Ok(()) } - fn next_tuple_owned(iter: &mut I) -> Result, DatabaseError> { - iter.next_tuple(|_, tuple| tuple.clone()) + async fn next_tuple_owned( + iter: &mut I, + ) -> Result, DatabaseError> { + iter.next_tuple(|_, tuple| tuple.clone()).await } - fn next_values(iter: &mut I) -> Result, DatabaseError> { - iter.next_tuple(|_, tuple| tuple.clone()) + async fn next_values(iter: &mut I) -> Result, DatabaseError> { + iter.next_tuple(|_, tuple| tuple.clone()).await } - fn read_single_i32(mut iter: I) -> Result + async fn read_single_i32(mut iter: I) -> Result where I: ResultIter, { - let value = match iter.next_tuple(|_, tuple| match (tuple.len(), tuple.get(0)) { - (1, Some(DataValue::Int32(value))) => *value, - _ => panic!("expected a single Int32 column, got {tuple:?}"), - })? { + let value = match iter + .next_tuple(|_, tuple| match (tuple.len(), tuple.get(0)) { + (1, Some(DataValue::Int32(value))) => *value, + _ => panic!("expected a single Int32 column, got {tuple:?}"), + }) + .await? + { Some(value) => value, None => panic!("expected one result row"), }; - iter.done()?; + iter.done().await?; Ok(value) } @@ -2042,28 +2221,35 @@ pub(crate) mod test { } #[test] - fn test_run_sql() -> Result<(), DatabaseError> { + async fn test_run_sql() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut database = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; - database.ddl("create table t1(c1 int primary key, c2 boolean, c3 int)")?; + let mut database = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; + database + .ddl("create table t1(c1 int primary key, c2 boolean, c3 int)") + .await?; - let mut iter = database.run("select * from t1")?; + let mut iter = database.run("select * from t1").await?; while iter .next_tuple(|_, tuple| { println!("{tuple:#?}"); - })? + }) + .await? .is_some() {} - iter.done()?; + iter.done().await?; Ok(()) } /// use [CurrentDate](crate::function::current_date::CurrentDate) on this case #[test] - fn test_udf() -> Result<(), DatabaseError> { + async fn test_udf() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; - let mut iter = kite_sql.run("select current_date()")?; + let kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; + let mut iter = kite_sql.run("select current_date()").await?; iter.schema(|schema| { assert_eq!(schema.len(), 1); @@ -2072,25 +2258,27 @@ pub(crate) mod test { assert_eq!(column.datatype(), &LogicalType::Date); }); assert_eq!( - next_tuple_owned(&mut iter)?.unwrap(), + next_tuple_owned(&mut iter).await?.unwrap(), Tuple::new( None, vec![DataValue::Date32(Local::now().num_days_from_ce())] ) ); - assert!(iter.next_tuple(|_, _| ())?.is_none()); + assert!(iter.next_tuple(|_, _| ()).await?.is_none()); Ok(()) } /// use [Numbers](crate::function::numbers::Numbers) on this case #[test] - fn test_udtf() -> Result<(), DatabaseError> { + async fn test_udtf() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; let mut iter = kite_sql.run( "SELECT * FROM (select * from table(numbers(10)) a ORDER BY number LIMIT 5) OFFSET 3", - )?; + ).await?; iter.schema(|schema| { assert_eq!(schema.len(), 1); @@ -2099,23 +2287,29 @@ pub(crate) mod test { assert_eq!(column.datatype(), &LogicalType::Integer); }); assert_eq!( - next_tuple_owned(&mut iter)?.unwrap(), + next_tuple_owned(&mut iter).await?.unwrap(), Tuple::new(None, vec![DataValue::Int32(3)]) ); assert_eq!( - next_tuple_owned(&mut iter)?.unwrap(), + next_tuple_owned(&mut iter).await?.unwrap(), Tuple::new(None, vec![DataValue::Int32(4)]) ); Ok(()) } #[test] - fn test_join_on_alias_right_key_is_localized() -> Result<(), DatabaseError> { + async fn test_join_on_alias_right_key_is_localized() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - kite_sql.ddl("CREATE TABLE onecolumn (id INT PRIMARY KEY, x INT NULL)")?; - kite_sql.ddl("CREATE TABLE empty (e_id INT PRIMARY KEY, x INT)")?; + kite_sql + .ddl("CREATE TABLE onecolumn (id INT PRIMARY KEY, x INT NULL)") + .await?; + kite_sql + .ddl("CREATE TABLE empty (e_id INT PRIMARY KEY, x INT)") + .await?; let stmt = kite_sql.state.parse_statement( "SELECT * FROM onecolumn AS a(aid, x) JOIN empty AS b(bid, y) ON a.x = b.y", @@ -2194,12 +2388,18 @@ pub(crate) mod test { } #[test] - fn test_join_on_with_right_filter_keeps_localized_key() -> Result<(), DatabaseError> { + async fn test_join_on_with_right_filter_keeps_localized_key() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - kite_sql.ddl("CREATE TABLE onecolumn (id INT PRIMARY KEY, x INT NULL)")?; - kite_sql.ddl("CREATE TABLE twocolumn (t_id INT PRIMARY KEY, x INT NULL, y INT NULL)")?; + kite_sql + .ddl("CREATE TABLE onecolumn (id INT PRIMARY KEY, x INT NULL)") + .await?; + kite_sql + .ddl("CREATE TABLE twocolumn (t_id INT PRIMARY KEY, x INT NULL, y INT NULL)") + .await?; let stmt = kite_sql.state.parse_statement( "SELECT o.x, t.y FROM onecolumn o INNER JOIN twocolumn t ON (o.x=t.x AND t.y=53)", @@ -2297,20 +2497,29 @@ pub(crate) mod test { } #[test] - fn test_join_on_with_right_filter_keeps_localized_key_with_data() -> Result<(), DatabaseError> { + async fn test_join_on_with_right_filter_keeps_localized_key_with_data( + ) -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - kite_sql.ddl("CREATE TABLE onecolumn (id INT PRIMARY KEY, x INT NULL)")?; - kite_sql.ddl("CREATE TABLE twocolumn (t_id INT PRIMARY KEY, x INT NULL, y INT NULL)")?; kite_sql - .run("INSERT INTO onecolumn(id, x) VALUES (0, 44), (1, NULL), (2, 42)")? - .done()?; + .ddl("CREATE TABLE onecolumn (id INT PRIMARY KEY, x INT NULL)") + .await?; + kite_sql + .ddl("CREATE TABLE twocolumn (t_id INT PRIMARY KEY, x INT NULL, y INT NULL)") + .await?; + kite_sql + .run("INSERT INTO onecolumn(id, x) VALUES (0, 44), (1, NULL), (2, 42)") + .await? + .done() + .await?; kite_sql .run( "INSERT INTO twocolumn(t_id, x, y) VALUES (0,44,51), (1,NULL,52), (2,42,53), (3,45,45)", - )? - .done()?; + ).await? + .done().await?; let stmt = kite_sql.state.parse_statement( "SELECT o.x, t.y FROM onecolumn o INNER JOIN twocolumn t ON (o.x=t.x AND t.y=53)", @@ -2385,17 +2594,32 @@ pub(crate) mod test { } #[test] - fn test_prepare_statment() -> Result<(), DatabaseError> { + async fn test_prepare_statment() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) .histogram_buckets(2) - .build_rocksdb()?; + .build_in_memory() + .await?; - kite_sql.ddl("create table t1 (a int primary key, b int)")?; - kite_sql.run("insert into t1 values(0, 0)")?.done()?; - kite_sql.run("insert into t1 values(1, 1)")?.done()?; - kite_sql.run("insert into t1 values(2, 2)")?.done()?; - kite_sql.analyze("t1")?; + kite_sql + .ddl("create table t1 (a int primary key, b int)") + .await?; + kite_sql + .run("insert into t1 values(0, 0)") + .await? + .done() + .await?; + kite_sql + .run("insert into t1 values(1, 1)") + .await? + .done() + .await?; + kite_sql + .run("insert into t1 values(2, 2)") + .await? + .done() + .await?; + kite_sql.analyze("t1").await?; let mut tx = kite_sql.new_transaction(false)?; // Filter @@ -2405,9 +2629,11 @@ pub(crate) mod test { &[(1, LogicalType::Integer)], )?; - let mut iter = statement.execute(&mut tx, &[(1, DataValue::Int32(0).into())])?; + let mut iter = statement + .execute(&mut tx, &[(1, DataValue::Int32(0).into())]) + .await?; - let row = next_tuple_owned(&mut iter)?.unwrap(); + let row = next_tuple_owned(&mut iter).await?.unwrap(); let plan = row[0].utf8().unwrap(); assert_eq!( plan, @@ -2425,16 +2651,18 @@ pub(crate) mod test { ], )?; - let mut iter = statement.execute( - &mut tx, - &[ - (1, DataValue::Int32(0).into()), - (2, DataValue::Int32(0).into()), - (3, DataValue::Int32(1).into()), - (4, DataValue::Int32(0).into()), - ], - )?; - let row = next_tuple_owned(&mut iter)?.unwrap(); + let mut iter = statement + .execute( + &mut tx, + &[ + (1, DataValue::Int32(0).into()), + (2, DataValue::Int32(0).into()), + (3, DataValue::Int32(1).into()), + (4, DataValue::Int32(0).into()), + ], + ) + .await?; + let row = next_tuple_owned(&mut iter).await?.unwrap(); let plan = row[0].utf8().unwrap(); assert_eq!( plan, @@ -2444,16 +2672,18 @@ pub(crate) mod test { { let mut statement = kite_sql.program_sql("explain select *, $1 from (select * from t1 where b > $2) as l left join (select * from t1 where a > $3) on l.a > $4", &[(1, LogicalType::Integer), (2, LogicalType::Integer), (3, LogicalType::Integer), (4, LogicalType::Integer)])?; - let mut iter = statement.execute( - &mut tx, - &[ - (1, DataValue::Int32(9).into()), - (2, DataValue::Int32(0).into()), - (3, DataValue::Int32(1).into()), - (4, DataValue::Int32(0).into()), - ], - )?; - let row = next_tuple_owned(&mut iter)?.unwrap(); + let mut iter = statement + .execute( + &mut tx, + &[ + (1, DataValue::Int32(9).into()), + (2, DataValue::Int32(0).into()), + (3, DataValue::Int32(1).into()), + (4, DataValue::Int32(0).into()), + ], + ) + .await?; + let row = next_tuple_owned(&mut iter).await?.unwrap(); let plan = row[0].utf8().unwrap(); assert_eq!( plan, @@ -2465,35 +2695,44 @@ pub(crate) mod test { } #[test] - fn test_run_multi_statement() -> Result<(), DatabaseError> { + async fn test_run_multi_statement() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - kite_sql.ddl("create table t_multi (a int primary key, b int)")?; + kite_sql + .ddl("create table t_multi (a int primary key, b int)") + .await?; let mut iter = kite_sql.run( "insert into t_multi values(0, 0); insert into t_multi values(1, 1); select * from t_multi order by a", - )?; + ).await?; assert_eq!( - next_values(&mut iter)?.unwrap(), + next_values(&mut iter).await?.unwrap(), vec![DataValue::Int32(0), DataValue::Int32(0)] ); assert_eq!( - next_values(&mut iter)?.unwrap(), + next_values(&mut iter).await?.unwrap(), vec![DataValue::Int32(1), DataValue::Int32(1)] ); - assert!(iter.next_tuple(|_, _| ())?.is_none()); - iter.done()?; + assert!(iter.next_tuple(|_, _| ()).await?.is_none()); + iter.done().await?; Ok(()) } #[test] - fn test_run_multi_statement_disallow_ddl() -> Result<(), DatabaseError> { + async fn test_run_multi_statement_disallow_ddl() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - let err = match kite_sql.run("create table t_multi_ddl (a int primary key); select 1") { + let err = match kite_sql + .run("create table t_multi_ddl (a int primary key); select 1") + .await + { Ok(_) => panic!("multi-statement execution with DDL should be rejected"), Err(err) => err, }; @@ -2508,13 +2747,20 @@ pub(crate) mod test { } #[test] - fn test_bind_error_with_span() -> Result<(), DatabaseError> { + async fn test_bind_error_with_span() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - kite_sql.ddl("create table t_bind_span(id int primary key)")?; + kite_sql + .ddl("create table t_bind_span(id int primary key)") + .await?; - let err = match kite_sql.run("select id, missing_col from t_bind_span") { + let err = match kite_sql + .run("select id, missing_col from t_bind_span") + .await + { Ok(_) => panic!("expected bind error"), Err(err) => err, }; @@ -2539,13 +2785,20 @@ pub(crate) mod test { } #[test] - fn test_bind_function_error_with_span() -> Result<(), DatabaseError> { + async fn test_bind_function_error_with_span() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - kite_sql.ddl("create table t_bind_fn_span(id int primary key)")?; + kite_sql + .ddl("create table t_bind_fn_span(id int primary key)") + .await?; - let err = match kite_sql.run("select missing_fn(id) from t_bind_fn_span") { + let err = match kite_sql + .run("select missing_fn(id) from t_bind_fn_span") + .await + { Ok(_) => panic!("expected function bind error"), Err(err) => err, }; @@ -2568,6 +2821,7 @@ pub(crate) mod test { Ok(()) } + #[cfg(feature = "rocksdb")] #[test] fn test_transaction_sql() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); @@ -2614,38 +2868,43 @@ pub(crate) mod test { } #[test] - fn test_transaction_run_multi_statement() -> Result<(), DatabaseError> { + async fn test_transaction_run_multi_statement() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - kite_sql.ddl("create table t_multi_tx (a int primary key, b int)")?; + kite_sql + .ddl("create table t_multi_tx (a int primary key, b int)") + .await?; let mut tx = kite_sql.new_transaction(false)?; let mut iter = tx.run( "insert into t_multi_tx values(0, 0); insert into t_multi_tx values(1, 1); select * from t_multi_tx order by a", - )?; + ).await?; assert_eq!( - next_values(&mut iter)?.unwrap(), + next_values(&mut iter).await?.unwrap(), vec![DataValue::Int32(0), DataValue::Int32(0)] ); assert_eq!( - next_values(&mut iter)?.unwrap(), + next_values(&mut iter).await?.unwrap(), vec![DataValue::Int32(1), DataValue::Int32(1)] ); - assert!(iter.next_tuple(|_, _| ())?.is_none()); - iter.done()?; - tx.commit()?; + assert!(iter.next_tuple(|_, _| ()).await?.is_none()); + iter.done().await?; + tx.commit().await?; - let mut check_iter = kite_sql.run("select count(*) from t_multi_tx")?; + let mut check_iter = kite_sql.run("select count(*) from t_multi_tx").await?; assert_eq!( - next_values(&mut check_iter)?.unwrap(), + next_values(&mut check_iter).await?.unwrap(), vec![DataValue::Int32(2)] ); - check_iter.done()?; + check_iter.done().await?; Ok(()) } + #[cfg(feature = "rocksdb")] #[test] fn test_autocommit_read_drops_iterator_before_transaction() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); @@ -2665,6 +2924,7 @@ pub(crate) mod test { Ok(()) } + #[cfg(feature = "rocksdb")] #[test] fn test_exhausted_database_iter_releases_mdl_guard() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); @@ -2692,6 +2952,7 @@ pub(crate) mod test { Ok(()) } + #[cfg(feature = "rocksdb")] #[test] fn test_rocksdb_defaults_to_repeatable_read() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); @@ -2705,6 +2966,7 @@ pub(crate) mod test { Ok(()) } + #[cfg(feature = "rocksdb")] #[test] fn test_repeatable_read_keeps_transaction_snapshot() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); @@ -2735,6 +2997,7 @@ pub(crate) mod test { Ok(()) } + #[cfg(feature = "rocksdb")] #[test] fn test_optimistic_transaction_sql() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); @@ -2786,11 +3049,12 @@ pub(crate) mod test { } #[test] - fn test_invalid_histogram_buckets() { + async fn test_invalid_histogram_buckets() { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); let result = DataBaseBuilder::path(temp_dir.path()) .histogram_buckets(0) - .build_rocksdb(); + .build_in_memory() + .await; assert!(matches!( result, @@ -2799,11 +3063,12 @@ pub(crate) mod test { } #[test] - fn test_invalid_statistics_relative_error() { + async fn test_invalid_statistics_relative_error() { for relative_error in [0.0, 1.0, -0.5, f64::NAN] { let result = DataBaseBuilder::path(".") .statistics_relative_error(relative_error) - .build_in_memory(); + .build_in_memory() + .await; assert!(matches!( result, Err(DatabaseError::InvalidValue(message)) @@ -2813,11 +3078,12 @@ pub(crate) mod test { } #[test] - fn test_invalid_hll_precision() { + async fn test_invalid_hll_precision() { for precision in [3, 27] { let result = DataBaseBuilder::path(".") .hll_precision(precision) - .build_in_memory(); + .build_in_memory() + .await; assert!(matches!( result, Err(DatabaseError::InvalidValue(message)) @@ -2827,54 +3093,69 @@ pub(crate) mod test { } #[test] - fn parser_recursion_limit_applies_to_sql_frontends() -> Result<(), DatabaseError> { + async fn parser_recursion_limit_applies_to_sql_frontends() -> Result<(), DatabaseError> { let expression = format!("{}7{}", "(".repeat(8), ")".repeat(8)); let sql = format!("select {expression}"); - let default = DataBaseBuilder::path(".").build_in_memory()?; - assert_eq!(read_single_i32(default.run(&sql)?)?, 7); + let default = DataBaseBuilder::path(".").build_in_memory().await?; + assert_eq!(read_single_i32(default.run(&sql).await?).await?, 7); assert_eq!(default.prepare_all(&sql)?.len(), 1); let mut db = DataBaseBuilder::path(".") .parser_recursion_limit(256) - .build_in_memory()?; + .build_in_memory() + .await?; assert_eq!(db.prepare_all(&sql)?.len(), 1); - assert_eq!(read_single_i32(db.run(&sql)?)?, 7); - db.ddl(format!("create view deep_view as {sql}"))?; - assert_eq!(read_single_i32(db.run("select * from deep_view")?)?, 7); - db.run_mut(&sql, |iter| { - assert_eq!( - iter.next_tuple(|_, row| row.clone())?.unwrap(), - vec![DataValue::Int32(7)] - ); - Ok(()) - })?; + assert_eq!(read_single_i32(db.run(&sql).await?).await?, 7); + db.ddl(format!("create view deep_view as {sql}")).await?; + assert_eq!( + read_single_i32(db.run("select * from deep_view").await?).await?, + 7 + ); + struct FirstRow; + impl crate::db::ConsumeMut for FirstRow { + type Output = TupleImpl; + + async fn consume<'a>( + self, + iter: &mut crate::db::TransactionIter< + 'a, + 'a, + S::TransactionType<'a>, + PlanArena<'a>, + >, + ) -> Result { + Ok(iter.next_tuple(|_, row| row.clone()).await?.unwrap()) + } + } + assert_eq!(db.run_mut(&sql, FirstRow).await?, vec![DataValue::Int32(7)]); let mut tx = db.new_transaction(false)?; let mut program = db.program_sql(&sql, &[])?; - let mut iter = program.execute(&mut tx, [])?; + let mut iter = program.execute(&mut tx, []).await?; assert_eq!( - iter.next_tuple(|_, row| row.clone())?.unwrap(), + iter.next_tuple(|_, row| row.clone()).await?.unwrap(), vec![DataValue::Int32(7)] ); - iter.done()?; - let mut iter = tx.run(&sql)?; + iter.done().await?; + let mut iter = tx.run(&sql).await?; assert_eq!( - iter.next_tuple(|_, row| row.clone())?.unwrap(), + iter.next_tuple(|_, row| row.clone()).await?.unwrap(), vec![DataValue::Int32(7)] ); - iter.done()?; + iter.done().await?; let mut program = tx.program_sql(&sql, &[])?; - let mut iter = program.execute(&mut tx, [])?; + let mut iter = program.execute(&mut tx, []).await?; assert_eq!( - iter.next_tuple(|_, row| row.clone())?.unwrap(), + iter.next_tuple(|_, row| row.clone()).await?.unwrap(), vec![DataValue::Int32(7)] ); - iter.done()?; - tx.commit()?; + iter.done().await?; + tx.commit().await?; let low = DataBaseBuilder::path(".") .parser_recursion_limit(4) - .build_in_memory()?; - assert!(low.run("select 1").is_ok()); + .build_in_memory() + .await?; + assert!(low.run("select 1").await.is_ok()); assert!(low .prepare_all(&sql) .unwrap_err() @@ -2882,12 +3163,13 @@ pub(crate) mod test { .contains("recursion limit exceeded")); assert!(low .run(&sql) + .await .err() .unwrap() .to_string() .contains("recursion limit exceeded")); assert!(matches!( - DataBaseBuilder::path(".").parser_recursion_limit(0).build_in_memory(), + DataBaseBuilder::path(".").parser_recursion_limit(0).build_in_memory().await, Err(DatabaseError::InvalidValue(message)) if message == "parser recursion limit must be >= 1" )); Ok(()) @@ -2926,6 +3208,7 @@ pub(crate) mod test { Ok(()) } + #[cfg(feature = "rocksdb")] #[test] fn optimistic_checkpoint_restores_rows_indexes_and_scalar_views() -> Result<(), DatabaseError> { let directory = TempDir::new()?; @@ -2994,6 +3277,7 @@ pub(crate) mod test { Ok(()) } + #[cfg(feature = "rocksdb")] #[test] fn test_checkpoint_rejects_non_empty_target_dir() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); @@ -3015,6 +3299,7 @@ pub(crate) mod test { Ok(()) } + #[cfg(feature = "rocksdb")] #[cfg(not(feature = "unsafe_txdb_checkpoint"))] #[test] fn test_checkpoint_requires_unsafe_feature() -> Result<(), DatabaseError> { diff --git a/src/db/program.rs b/src/db/program.rs index fd643469..a922f219 100644 --- a/src/db/program.rs +++ b/src/db/program.rs @@ -65,10 +65,11 @@ pub struct Program<'p> { pub(crate) arena: ExecMetaArena>, } +#[maybe_async] impl<'p> Program<'p> { /// Runs the program with `params` in `transaction`. The returned iterator borrows the /// program until the execution ends. - pub fn execute<'a, 'txn, 'v, S: Storage>( + pub async fn execute<'a, 'txn, 'v, S: Storage>( &'a mut self, transaction: &'a mut Transaction<'txn, S>, params: impl AsRef<[(usize, Param<'v>)]>, @@ -88,7 +89,8 @@ impl<'p> Program<'p> { &mut transaction.inner, &mut self.executor, &mut self.arena, - )?; + ) + .await?; Ok(ProgramIter { guard: TransactionGuard { arena, @@ -206,6 +208,7 @@ pub struct ProgramIter<'a, 'p, T: StorageTransaction + 'p> { program: &'a mut Program<'p>, } +#[maybe_async] impl<'a, 'p, T: StorageTransaction + 'p> ProgramIter<'a, 'p, T> { #[inline] pub fn schema(&self, f: impl FnOnce(&SchemaView<'_, '_>) -> R) -> R { @@ -214,17 +217,17 @@ impl<'a, 'p, T: StorageTransaction + 'p> ProgramIter<'a, 'p, T> { } #[inline] - pub fn next_tuple( + pub async fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R + MaybeSend, ) -> Result, DatabaseError> { let Program { executor, arena } = &mut *self.program; - self.guard.next_tuple(executor, arena, f) + self.guard.next_tuple(executor, arena, f).await } #[inline] - pub fn done(mut self) -> Result<(), DatabaseError> { - while self.next_tuple(|_, _| ())?.is_some() {} + pub async fn done(mut self) -> Result<(), DatabaseError> { + while self.next_tuple(|_, _| ()).await?.is_some() {} Ok(()) } } @@ -235,34 +238,37 @@ impl Drop for ProgramIter<'_, '_, T> { } } +#[maybe_async] impl<'a, 'p, T: StorageTransaction + 'p> TxnResultIter<'a> for ProgramIter<'a, 'p, T> { - fn next_txn_tuple( + async fn next_txn_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, 'a>) -> R + MaybeSend, ) -> Result, DatabaseError> { - ProgramIter::next_tuple(self, f) + ProgramIter::next_tuple(self, f).await } } +#[maybe_async] impl<'p, T: StorageTransaction + 'p> ResultIter for ProgramIter<'_, 'p, T> { fn schema(&self, f: impl FnOnce(&SchemaView<'_, '_>) -> R) -> R { ProgramIter::schema(self, f) } - fn next_tuple( + async fn next_tuple( &mut self, - f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R, + f: impl FnOnce(&SchemaView<'_, '_>, &mut ResultRow<'_, '_>) -> R + MaybeSend, ) -> Result, DatabaseError> { - ProgramIter::next_tuple(self, f) + ProgramIter::next_tuple(self, f).await } - fn done(self) -> Result<(), DatabaseError> { - ProgramIter::done(self) + async fn done(self) -> Result<(), DatabaseError> { + ProgramIter::done(self).await } } // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; @@ -281,15 +287,17 @@ mod tests { use std::ops::Bound; #[test] - fn prepare_caches_scalar_output_schema() -> Result<(), DatabaseError> { - let db = DataBaseBuilder::path(".").build_in_memory()?; + async fn prepare_caches_scalar_output_schema() -> Result<(), DatabaseError> { + let db = DataBaseBuilder::path(".").build_in_memory().await?; let mut plan = db.program_sql( "select (($1 * 3 + 7) % 97) + ($1 / 2)", &[(1, LogicalType::Bigint)], )?; let mut tx = db.new_transaction(false)?; for (value, expected) in [(7, 31_i64), (23, 87_i64)] { - let mut iter = plan.execute(&mut tx, [(1, DataValue::Int64(value).into())])?; + let mut iter = plan + .execute(&mut tx, [(1, DataValue::Int64(value).into())]) + .await?; iter.schema(|schema| { assert_eq!(schema.len(), 1); assert_eq!( @@ -298,24 +306,26 @@ mod tests { ); }); assert_eq!( - iter.next_tuple(|_, row| row.clone())?.unwrap(), + iter.next_tuple(|_, row| row.clone()).await?.unwrap(), vec![DataValue::Int64(expected)] ); - assert!(iter.next_tuple(|_, _| ())?.is_none()); - iter.done()?; + assert!(iter.next_tuple(|_, _| ()).await?.is_none()); + iter.done().await?; } - let mut iter = plan.execute(&mut tx, [(1, DataValue::Int64(7).into())])?; + let mut iter = plan + .execute(&mut tx, [(1, DataValue::Int64(7).into())]) + .await?; assert_eq!( - iter.next_tuple(|_, row| row.clone())?.unwrap(), + iter.next_tuple(|_, row| row.clone()).await?.unwrap(), vec![DataValue::Int64(31)] ); - iter.done()?; - tx.commit()?; + iter.done().await?; + tx.commit().await?; Ok(()) } #[test] - fn specialize_selected_index_and_preserve_plan_metadata() -> Result<(), DatabaseError> { + async fn specialize_selected_index_and_preserve_plan_metadata() -> Result<(), DatabaseError> { let table_arena = TableArenaCell::default(); let mut arena = PlanArena::new(&table_arena); let mut column = ColumnCatalog::new( @@ -349,7 +359,7 @@ mod tests { }; let tables = TableCache::default(); let context = &tables; - let storage = MemoryStorage::new(); + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; for (lookup, has_residual, parameter, should_change) in [ ( @@ -504,7 +514,7 @@ mod tests { } #[test] - fn parameter_order_dependent_predicates_remain_correct() -> Result<(), DatabaseError> { + async fn parameter_order_dependent_predicates_remain_correct() -> Result<(), DatabaseError> { fn has_filter(plan: &LogicalPlan) -> bool { matches!( &plan.operator, @@ -518,11 +528,15 @@ mod tests { let mut db = DataBaseBuilder::path(".") .histogram_buckets(2) - .build_in_memory()?; - db.ddl("create table parameter_order(id int primary key)")?; - db.run("insert into parameter_order values(1),(2),(3),(4)")? - .done()?; - db.analyze("parameter_order")?; + .build_in_memory() + .await?; + db.ddl("create table parameter_order(id int primary key)") + .await?; + db.run("insert into parameter_order values(1),(2),(3),(4)") + .await? + .done() + .await?; + db.analyze("parameter_order").await?; let mut tx = db.new_transaction(false)?; let mut lower_bounds = db.program_sql( @@ -530,48 +544,55 @@ mod tests { &[(1, LogicalType::Integer), (2, LogicalType::Integer)], )?; assert!(has_filter(lower_bounds.executor.plan())); - let mut iter = lower_bounds.execute( - &mut tx, - [ - (1, DataValue::Int32(3).into()), - (2, DataValue::Int32(1).into()), - ], - )?; + let mut iter = lower_bounds + .execute( + &mut tx, + [ + (1, DataValue::Int32(3).into()), + (2, DataValue::Int32(1).into()), + ], + ) + .await?; let mut rows = Vec::new(); while iter - .next_tuple(|_, row| rows.push(row.value_at(0).clone()))? + .next_tuple(|_, row| rows.push(row.value_at(0).clone())) + .await? .is_some() {} - iter.done()?; + iter.done().await?; assert_eq!(rows, vec![DataValue::Int32(3), DataValue::Int32(4)]); let mut equalities = db.program_sql( "select id from parameter_order where id = $1 and id = $2", &[(1, LogicalType::Integer), (2, LogicalType::Integer)], )?; - let mut iter = equalities.execute( - &mut tx, - [ - (1, DataValue::Int32(2).into()), - (2, DataValue::Int32(2).into()), - ], - )?; + let mut iter = equalities + .execute( + &mut tx, + [ + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(2).into()), + ], + ) + .await?; assert_eq!( - iter.next_tuple(|_, row| row.value_at(0).clone())?, + iter.next_tuple(|_, row| row.value_at(0).clone()).await?, Some(DataValue::Int32(2)) ); - assert!(iter.next_tuple(|_, _| ())?.is_none()); - iter.done()?; - - let mut iter = equalities.execute( - &mut tx, - [ - (1, DataValue::Int32(2).into()), - (2, DataValue::Int32(3).into()), - ], - )?; - assert!(iter.next_tuple(|_, _| ())?.is_none()); - iter.done()?; + assert!(iter.next_tuple(|_, _| ()).await?.is_none()); + iter.done().await?; + + let mut iter = equalities + .execute( + &mut tx, + [ + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(3).into()), + ], + ) + .await?; + assert!(iter.next_tuple(|_, _| ()).await?.is_none()); + iter.done().await?; let mut unions = db.program_sql( "select id from parameter_order where (id=$1 or id=$2) and id>$3 order by id", @@ -613,24 +634,27 @@ mod tests { vec![], ), ] { - let mut iter = unions.execute( - &mut tx, - [(1, first.into()), (2, second.into()), (3, lower.into())], - )?; + let mut iter = unions + .execute( + &mut tx, + [(1, first.into()), (2, second.into()), (3, lower.into())], + ) + .await?; let mut rows = Vec::new(); while iter - .next_tuple(|_, row| rows.push(row.value_at(0).clone()))? + .next_tuple(|_, row| rows.push(row.value_at(0).clone())) + .await? .is_some() {} - iter.done()?; + iter.done().await?; assert_eq!(rows, expected); } Ok(()) } #[test] - fn repeated_execution_and_null_do_not_stale_parameters() -> Result<(), DatabaseError> { - let db = DataBaseBuilder::path(".").build_in_memory()?; + async fn repeated_execution_and_null_do_not_stale_parameters() -> Result<(), DatabaseError> { + let db = DataBaseBuilder::path(".").build_in_memory().await?; let mut plan = db.program_sql( "values ($1 + $2), ($2)", &[(1, LogicalType::Integer), (2, LogicalType::Integer)], @@ -640,38 +664,42 @@ mod tests { (DataValue::Null, DataValue::Null), (DataValue::Int32(3), DataValue::Int32(13)), ] { - let mut iter = plan.execute( - &mut tx, - [(1, DataValue::Int32(10).into()), (2, value.clone().into())], - )?; + let mut iter = plan + .execute( + &mut tx, + [(1, DataValue::Int32(10).into()), (2, value.clone().into())], + ) + .await?; assert_eq!( - iter.next_tuple(|_, row| row.value_at(0).clone())?, + iter.next_tuple(|_, row| row.value_at(0).clone()).await?, Some(expected) ); assert_eq!( - iter.next_tuple(|_, row| row.value_at(0).clone())?, + iter.next_tuple(|_, row| row.value_at(0).clone()).await?, Some(value) ); - assert!(iter.next_tuple(|_, _| ())?.is_none()); - iter.done()?; + assert!(iter.next_tuple(|_, _| ()).await?.is_none()); + iter.done().await?; } assert!(matches!( - plan.execute(&mut tx, [(1, DataValue::Int32(10).into())]), + plan.execute(&mut tx, [(1, DataValue::Int32(10).into())]).await, Err(DatabaseError::ParametersNotFound { name, .. }) if name == "$2" )); - let mut iter = plan.execute( - &mut tx, - [ - (1, DataValue::Int32(10).into()), - (2, DataValue::Int32(4).into()), - ], - )?; + let mut iter = plan + .execute( + &mut tx, + [ + (1, DataValue::Int32(10).into()), + (2, DataValue::Int32(4).into()), + ], + ) + .await?; assert_eq!( - iter.next_tuple(|_, row| row.value_at(0).clone())?, + iter.next_tuple(|_, row| row.value_at(0).clone()).await?, Some(DataValue::Int32(14)) ); - iter.done()?; + iter.done().await?; Ok(()) } @@ -690,14 +718,18 @@ mod tests { } #[test] - fn null_parameters_empty_index_ranges() -> Result<(), DatabaseError> { + async fn null_parameters_empty_index_ranges() -> Result<(), DatabaseError> { let mut db = DataBaseBuilder::path(".") .histogram_buckets(2) - .build_in_memory()?; - db.ddl("create table t(w int, k int, primary key(w,k))")?; - db.run("insert into t values(1,1),(1,2),(2,1),(2,2),(2,4)")? - .done()?; - db.analyze("t")?; + .build_in_memory() + .await?; + db.ddl("create table t(w int, k int, primary key(w,k))") + .await?; + db.run("insert into t values(1,1),(1,2),(2,1),(2,2),(2,4)") + .await? + .done() + .await?; + db.analyze("t").await?; let ty = LogicalType::Integer; let mut tx = db.new_transaction(false)?; // A NULL bound used to reach storage as a NULL key: the composite scope below became @@ -727,23 +759,27 @@ mod tests { .collect::>(), )?; assert!(index_range(plan.executor.plan()).is_some(), "{sql}"); - let mut iter = plan.execute(&mut tx, ¶ms)?; - assert!(iter.next_tuple(|_, _| ())?.is_none(), "{sql}"); - iter.done()?; + let mut iter = plan.execute(&mut tx, ¶ms).await?; + assert!(iter.next_tuple(|_, _| ()).await?.is_none(), "{sql}"); + iter.done().await?; } Ok(()) } #[test] - fn composite_index_ranges_bind_for_each_execution() -> Result<(), DatabaseError> { + async fn composite_index_ranges_bind_for_each_execution() -> Result<(), DatabaseError> { let mut db = DataBaseBuilder::path(".") .histogram_buckets(2) - .build_in_memory()?; - db.ddl("create table t(w int, k int, primary key(w,k))")?; - db.run("insert into t values(1,1),(1,2),(1,3),(2,1),(2,2),(2,4)")? - .done()?; - db.analyze("t")?; + .build_in_memory() + .await?; + db.ddl("create table t(w int, k int, primary key(w,k))") + .await?; + db.run("insert into t values(1,1),(1,2),(1,3),(2,1),(2,2),(2,4)") + .await? + .done() + .await?; + db.analyze("t").await?; let mut tx = db.new_transaction(false)?; let mut plan = db.program_sql( "select k from t where w=$1 and k >= $2 and k < $3 order by k", @@ -754,35 +790,40 @@ mod tests { ], )?; assert!(index_range(plan.executor.plan()).is_some()); - let mut iter = plan.execute( - &mut tx, - [ - (1, DataValue::Int32(2).into()), - (2, DataValue::Int32(2).into()), - (3, DataValue::Int32(5).into()), - ], - )?; + let mut iter = plan + .execute( + &mut tx, + [ + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(2).into()), + (3, DataValue::Int32(5).into()), + ], + ) + .await?; let mut rows = Vec::new(); while iter - .next_tuple(|_, row| rows.push(row.value_at(0).clone()))? + .next_tuple(|_, row| rows.push(row.value_at(0).clone())) + .await? .is_some() {} - iter.done()?; + iter.done().await?; assert_eq!(rows, vec![DataValue::Int32(2), DataValue::Int32(4)]); - let mut iter = plan.execute( - &mut tx, - [ - (1, DataValue::Int32(1).into()), - (2, DataValue::Int32(1).into()), - (3, DataValue::Int32(2).into()), - ], - )?; + let mut iter = plan + .execute( + &mut tx, + [ + (1, DataValue::Int32(1).into()), + (2, DataValue::Int32(1).into()), + (3, DataValue::Int32(2).into()), + ], + ) + .await?; assert_eq!( - iter.next_tuple(|_, row| row.value_at(0).clone())?, + iter.next_tuple(|_, row| row.value_at(0).clone()).await?, Some(DataValue::Int32(1)) ); - assert!(iter.next_tuple(|_, _| ())?.is_none()); - iter.done()?; + assert!(iter.next_tuple(|_, _| ()).await?.is_none()); + iter.done().await?; // The lower bound is computed only after binding; it must still be // intersected with the prepared upper bound instead of scanning the @@ -838,7 +879,7 @@ mod tests { assert!(!has_filter(plan.executor.plan())); let tables = TableCache::default(); let context = &tables; - let storage = MemoryStorage::new(); + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let mut exec_buffer = crate::execution::ExecBuffer::default(); let mut exec_arena = crate::execution::ExecArena::new(context, &transaction); @@ -871,16 +912,18 @@ mod tests { (DataValue::Int32(24), vec![4]), (DataValue::Null, vec![]), ] { - let mut iter = plan.execute( - &mut tx, - [ - (1, DataValue::Int32(2).into()), - (2, DataValue::Int32(5).into()), - (3, lower.into()), - ], - )?; + let mut iter = plan + .execute( + &mut tx, + [ + (1, DataValue::Int32(2).into()), + (2, DataValue::Int32(5).into()), + (3, lower.into()), + ], + ) + .await?; let mut rows = Vec::new(); - while let Some(k) = iter.next_tuple(|_, row| row.value_at(0).clone())? { + while let Some(k) = iter.next_tuple(|_, row| row.value_at(0).clone()).await? { rows.push(k); } let expected = expected @@ -893,12 +936,16 @@ mod tests { } #[test] - fn rerun_rebuilds_materialized_subquery() -> Result<(), DatabaseError> { - let mut db = DataBaseBuilder::path(".").build_in_memory()?; - db.ddl("create table t1 (c1 int primary key, name varchar)")?; - db.ddl("create table t2 (c1 int primary key, name varchar)")?; - db.run("insert into t1 values (0, 'a'), (1, 'b'), (2, 'c'), (3, 'd')")? - .done()?; + async fn rerun_rebuilds_materialized_subquery() -> Result<(), DatabaseError> { + let mut db = DataBaseBuilder::path(".").build_in_memory().await?; + db.ddl("create table t1 (c1 int primary key, name varchar)") + .await?; + db.ddl("create table t2 (c1 int primary key, name varchar)") + .await?; + db.run("insert into t1 values (0, 'a'), (1, 'b'), (2, 'c'), (3, 'd')") + .await? + .done() + .await?; let mut plan = db.program_sql("select c1 from t1 where name in (select name from t2)", &[])?; @@ -907,15 +954,15 @@ mod tests { ("insert into t2 values (0, 'b')", vec![1]), ("insert into t2 values (1, 'd')", vec![1, 3]), ] { - db.run(insert)?.done()?; + db.run(insert).await?.done().await?; let mut tx = db.new_transaction(true)?; - let mut iter = plan.execute(&mut tx, [])?; + let mut iter = plan.execute(&mut tx, []).await?; let mut rows = Vec::new(); - while let Some(c1) = iter.next_tuple(|_, row| row.value_at(0).clone())? { + while let Some(c1) = iter.next_tuple(|_, row| row.value_at(0).clone()).await? { rows.push(c1); } - iter.done()?; - tx.commit()?; + iter.done().await?; + tx.commit().await?; let expected = expected .into_iter() .map(DataValue::Int32) diff --git a/src/errors.rs b/src/errors.rs index 17ab6415..1e619149 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -129,6 +129,10 @@ pub enum DatabaseError { Lmdb(lmdb::Error), #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] RocksDB(rocksdb::Error), + #[cfg(feature = "slatedb")] + SlateDB(slatedb::Error), + #[cfg(feature = "slatedb")] + ObjectStore(slatedb::object_store::Error), SharedNotAlign, SourceNotFound, TableExists, @@ -237,6 +241,10 @@ impl fmt::Display for DatabaseError { Self::Lmdb(err) => write!(f, "lmdb: {err}"), #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] Self::RocksDB(err) => write!(f, "rocksdb: {err}"), + #[cfg(feature = "slatedb")] + Self::SlateDB(err) => write!(f, "slatedb: {err}"), + #[cfg(feature = "slatedb")] + Self::ObjectStore(err) => write!(f, "object store: {err}"), Self::SharedNotAlign => { f.write_str("the number of caches cannot be divisible by the number of shards") } @@ -296,6 +304,10 @@ impl Error for DatabaseError { Self::Lmdb(err) => Some(err), #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] Self::RocksDB(err) => Some(err), + #[cfg(feature = "slatedb")] + Self::SlateDB(err) => Some(err), + #[cfg(feature = "slatedb")] + Self::ObjectStore(err) => Some(err), #[cfg(feature = "decimal")] Self::TryFromDecimal(err) => Some(err), Self::TryFromInt(err) => Some(err), @@ -330,6 +342,10 @@ impl_from_database_error!(ParserError, ParserSql); impl_from_database_error!(lmdb::Error, Lmdb); #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] impl_from_database_error!(rocksdb::Error, RocksDB); +#[cfg(feature = "slatedb")] +impl_from_database_error!(slatedb::Error, SlateDB); +#[cfg(feature = "slatedb")] +impl_from_database_error!(slatedb::object_store::Error, ObjectStore); #[cfg(feature = "decimal")] impl_from_database_error!(rust_decimal::Error, TryFromDecimal); impl_from_database_error!(TryFromIntError, TryFromInt); @@ -342,6 +358,19 @@ impl From for DatabaseError { } impl DatabaseError { + pub fn is_conflict(&self) -> bool { + match self { + #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] + Self::RocksDB(err) => matches!( + err.kind(), + rocksdb::ErrorKind::Busy | rocksdb::ErrorKind::TryAgain + ), + #[cfg(feature = "slatedb")] + Self::SlateDB(err) => err.kind() == slatedb::ErrorKind::Transaction, + _ => false, + } + } + pub fn invalid_column(name: impl Into) -> Self { Self::InvalidColumn { name: name.into(), @@ -525,6 +554,7 @@ fn build_sql_highlight(sql: &str, span: &SqlErrorSpan) -> Option { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; use crate::expression::{BinaryOperator, UnaryOperator}; diff --git a/src/execution/ddl/add_column.rs b/src/execution/ddl/add_column.rs index daef8966..3741bbaf 100644 --- a/src/execution/ddl/add_column.rs +++ b/src/execution/ddl/add_column.rs @@ -38,8 +38,9 @@ impl<'a> WriteExecutor<'a> for AddColumn<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for AddColumn<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -47,7 +48,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for AddColumn<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -81,13 +82,15 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for AddColumn<'a> { let (unique_index_id, apply) = { let (transaction, table_codec) = arena.transaction_codec_mut(); - let (table, col_id) = transaction.add_column( - table_codec, - plan_arena, - table_catalog.name(), - column, - *if_not_exists, - )?; + let (table, col_id) = transaction + .add_column( + table_codec, + plan_arena, + table_catalog.name(), + column, + *if_not_exists, + ) + .await?; let unique_meta = if column.desc().is_unique() { table .get_unique_index(&col_id, plan_arena) @@ -106,12 +109,14 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for AddColumn<'a> { .collect_vec(); serializers.push(column.datatype().serializable()); + let table_id = *table_id; + let index_default = default_value.clone(); let mut state = arena.local_state(plan_arena); let (transaction, table_codec) = state.transaction_codec_mut(); rewrite_table_in_batches( transaction, table_codec, - *table_id, + table_id, &pk_ty, &serializers[..old_schema.len()], &serializers, @@ -123,20 +128,23 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for AddColumn<'a> { } Ok(()) }, - |transaction, table_codec, tuple| { + async move |transaction, table_codec, tuple| { if let (Some(unique_index_id), Some(value), Some(tuple_id)) = - (unique_index_id.as_ref(), default_value.as_ref(), tuple.pk()) + (unique_index_id.as_ref(), index_default.as_ref(), tuple.pk()) { let index = Index::new( *unique_index_id, std::slice::from_ref(value), IndexType::Unique, ); - transaction.add_index(table_codec, *table_id, index, tuple_id)?; + transaction + .add_index(table_codec, table_id, index, tuple_id) + .await?; } Ok(()) }, - )?; + ) + .await?; arena.finish(); Ok(()) diff --git a/src/execution/ddl/change_column.rs b/src/execution/ddl/change_column.rs index b93bc8fd..eeb85c6a 100644 --- a/src/execution/ddl/change_column.rs +++ b/src/execution/ddl/change_column.rs @@ -37,8 +37,9 @@ impl<'a> WriteExecutor<'a> for ChangeColumn<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ChangeColumn<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -46,7 +47,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ChangeColumn<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -119,6 +120,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ChangeColumn<'a> { new_serializers[column_index] = data_type.serializable(); let mut state = arena.local_state(plan_arena); + let (transaction, table_codec) = state.transaction_codec_mut(); rewrite_table_in_batches( transaction, @@ -135,8 +137,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ChangeColumn<'a> { } Ok(()) }, - |_, _, _| Ok(()), - )?; + async |_, _, _| Ok(()), + ) + .await?; } else if needs_not_null_validation { let target_column_name = new_column_name.clone(); let old_serializers = old_serializers(); @@ -154,21 +157,24 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ChangeColumn<'a> { } Ok(()) }, - )?; + ) + .await?; } let apply = { let (transaction, table_codec) = arena.transaction_codec_mut(); - let table = transaction.change_column( - table_codec, - plan_arena, - table_catalog.name(), - old_column_name, - new_column_name, - data_type, - default_change, - not_null_change, - )?; + let table = transaction + .change_column( + table_codec, + plan_arena, + table_catalog.name(), + old_column_name, + new_column_name, + data_type, + default_change, + not_null_change, + ) + .await?; DDLApply::upsert_table(table, true) }; arena.push_ddl_apply(apply); diff --git a/src/execution/ddl/create_index.rs b/src/execution/ddl/create_index.rs index def00591..ba65d588 100644 --- a/src/execution/ddl/create_index.rs +++ b/src/execution/ddl/create_index.rs @@ -50,16 +50,17 @@ impl<'a> WriteExecutor<'a> for CreateIndex<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CreateIndex<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -99,14 +100,16 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CreateIndex<'a> { .unzip(); let index_id_result = { let (transaction, table_codec) = arena.transaction_codec_mut(); - let (table, index_id) = transaction.add_index_meta( - table_codec, - plan_arena, - table_catalog.name(), - index_name.clone(), - column_ids, - *ty, - )?; + let (table, index_id) = transaction + .add_index_meta( + table_codec, + plan_arena, + table_catalog.name(), + index_name.clone(), + column_ids, + *ty, + ) + .await?; let table_id = table.id(); arena.push_ddl_apply(DDLApply::upsert_table(table, false)); Ok((table_id, index_id)) @@ -124,7 +127,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CreateIndex<'a> { Err(err) => return Err(err), }; - while self.input.next_tuple(arena, plan_arena)? { + while self.input.next_tuple(arena, plan_arena).await? { if arena.result_tuple().pk().is_none() { continue; } @@ -134,7 +137,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CreateIndex<'a> { let (tuple, transaction, table_codec) = state.tuple_transaction_codec_mut(); let tuple_pk = tuple.pk().ok_or(DatabaseError::PrimaryKeyNotFound)?; let index = Index::new(index_id, tuple, *ty); - transaction.add_index(table_codec, table_id, index, tuple_pk)?; + transaction + .add_index(table_codec, table_id, index, tuple_pk) + .await?; } } diff --git a/src/execution/ddl/create_table.rs b/src/execution/ddl/create_table.rs index e01c66b4..cd5b7d29 100644 --- a/src/execution/ddl/create_table.rs +++ b/src/execution/ddl/create_table.rs @@ -34,8 +34,9 @@ impl<'a> WriteExecutor<'a> for CreateTable<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CreateTable<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -43,7 +44,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CreateTable<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -55,13 +56,15 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CreateTable<'a> { } = self.op; let (transaction, table_codec) = arena.transaction_codec_mut(); - let table = transaction.create_table( - table_codec, - plan_arena, - table_name.clone(), - columns.clone(), - *if_not_exists, - )?; + let table = transaction + .create_table( + table_codec, + plan_arena, + table_name.clone(), + columns.clone(), + *if_not_exists, + ) + .await?; if let Some(table) = table { arena.push_ddl_apply(DDLApply::upsert_table(table, false)); } diff --git a/src/execution/ddl/create_view.rs b/src/execution/ddl/create_view.rs index 3be25aa9..357423e5 100644 --- a/src/execution/ddl/create_view.rs +++ b/src/execution/ddl/create_view.rs @@ -34,8 +34,9 @@ impl<'a> WriteExecutor<'a> for CreateView<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CreateView<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -43,14 +44,16 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CreateView<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let CreateViewOperator { view, or_replace } = self.op; let (transaction, table_codec) = arena.transaction_codec_mut(); - let view_id = transaction.create_view(table_codec, plan_arena, view, *or_replace)?; + let view_id = transaction + .create_view(table_codec, plan_arena, view, *or_replace) + .await?; arena.push_ddl_apply(DDLApply::upsert_view(view_id, view.clone())); arena.finish(); diff --git a/src/execution/ddl/drop_column.rs b/src/execution/ddl/drop_column.rs index f373336b..9b234297 100644 --- a/src/execution/ddl/drop_column.rs +++ b/src/execution/ddl/drop_column.rs @@ -36,8 +36,9 @@ impl<'a> WriteExecutor<'a> for DropColumn<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropColumn<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -45,7 +46,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropColumn<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -89,6 +90,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropColumn<'a> { new_serializers.remove(column_index); let mut state = arena.local_state(plan_arena); + let (transaction, table_codec) = state.transaction_codec_mut(); rewrite_table_in_batches( transaction, @@ -101,17 +103,15 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropColumn<'a> { let _ = tuple.remove(column_index); Ok(()) }, - |_, _, _| Ok(()), - )?; + async |_, _, _| Ok(()), + ) + .await?; } { let (transaction, table_codec) = arena.transaction_codec_mut(); - let table = transaction.drop_column( - table_codec, - plan_arena, - table_catalog.name(), - column_name, - )?; + let table = transaction + .drop_column(table_codec, plan_arena, table_catalog.name(), column_name) + .await?; arena.push_ddl_apply(DDLApply::upsert_table(table, true)); } diff --git a/src/execution/ddl/drop_index.rs b/src/execution/ddl/drop_index.rs index 849faebb..0f8c245d 100644 --- a/src/execution/ddl/drop_index.rs +++ b/src/execution/ddl/drop_index.rs @@ -34,8 +34,9 @@ impl<'a> WriteExecutor<'a> for DropIndex<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropIndex<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -43,7 +44,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropIndex<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -56,7 +57,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropIndex<'a> { let dropped = { let (transaction, table_codec) = arena.transaction_codec_mut(); - transaction.drop_index(table_codec, plan_arena, table_name, index_name, *if_exists)? + transaction + .drop_index(table_codec, plan_arena, table_name, index_name, *if_exists) + .await? }; if let Some((table, index_id)) = dropped { let table_id = table.id(); diff --git a/src/execution/ddl/drop_table.rs b/src/execution/ddl/drop_table.rs index 0dbc7102..5b598720 100644 --- a/src/execution/ddl/drop_table.rs +++ b/src/execution/ddl/drop_table.rs @@ -34,8 +34,9 @@ impl<'a> WriteExecutor<'a> for DropTable<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropTable<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -43,7 +44,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropTable<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -54,7 +55,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropTable<'a> { } = self.op; let (transaction, table_codec) = arena.transaction_codec_mut(); - if transaction.drop_table(table_codec, plan_arena, table_name.clone(), *if_exists)? { + if transaction + .drop_table(table_codec, plan_arena, table_name.clone(), *if_exists) + .await? + { arena.push_ddl_apply(DDLApply::DropTable { name: table_name.clone(), }); diff --git a/src/execution/ddl/drop_view.rs b/src/execution/ddl/drop_view.rs index f1ca9a56..1c1e4e86 100644 --- a/src/execution/ddl/drop_view.rs +++ b/src/execution/ddl/drop_view.rs @@ -34,8 +34,9 @@ impl<'a> WriteExecutor<'a> for DropView<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropView<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -43,7 +44,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropView<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -54,7 +55,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for DropView<'a> { } = self.op; let (transaction, table_codec) = arena.transaction_codec_mut(); - if transaction.drop_view(table_codec, plan_arena, view_name.clone(), *if_exists)? { + if transaction + .drop_view(table_codec, plan_arena, view_name.clone(), *if_exists) + .await? + { arena.push_ddl_apply(DDLApply::DropView { name: view_name.clone(), }); diff --git a/src/execution/ddl/mod.rs b/src/execution/ddl/mod.rs index ff941ae5..2107440d 100644 --- a/src/execution/ddl/mod.rs +++ b/src/execution/ddl/mod.rs @@ -34,8 +34,9 @@ use std::collections::Bound; const REWRITE_BATCH_SIZE: usize = 1024; +#[maybe_async] #[allow(clippy::too_many_arguments)] -fn read_tuple_batch( +async fn read_tuple_batch( transaction: &T, table_codec: &mut TableCodec, table_id: TableId, @@ -46,58 +47,63 @@ fn read_tuple_batch( batch_size: usize, ) -> Result { let lower = if let Some(last_pk) = start_after { - table_codec.with_tuple_unchecked(table_id, last_pk, None, |key, _, _| { - Ok::<_, DatabaseError>(Bound::Excluded(key.to_vec())) - })? + table_codec + .with_tuple_unchecked(table_id, last_pk, None, async |key, _, _| { + Ok::<_, DatabaseError>(Bound::Excluded(key.to_vec())) + }) + .await? } else { Bound::Unbounded }; - table_codec.with_tuple_bound(table_id, |min, max, pool| { - let lower = match &lower { - Bound::Included(bytes) => Bound::Included(bytes.as_slice()), - Bound::Excluded(bytes) => Bound::Excluded(bytes.as_slice()), - Bound::Unbounded => Bound::Included(min), - }; - let mut iter = transaction.range(lower, Bound::Included(max), pool)?; - let mut len = 0; - - while len < batch_size { - let Some((key, value)) = iter.try_next()? else { - break; + table_codec + .with_tuple_bound(table_id, async |min, max, pool| { + let lower = match &lower { + Bound::Included(bytes) => Bound::Included(bytes.as_slice()), + Bound::Excluded(bytes) => Bound::Excluded(bytes.as_slice()), + Bound::Unbounded => Bound::Included(min), }; - let tuple_id = TableCodec::decode_tuple_key(key, pk_ty)?; - let tuple = if len < batch.len() { - &mut batch[len] - } else { - batch.push(TupleImpl::from(Tuple::new( - None, - Vec::with_capacity(old_deserializers.len()), - ))); - batch - .last_mut() - .expect("batch contains the tuple that was just pushed") - }; - // SAFETY: without a scope every value is copied. - unsafe { - TableCodec::decode_tuple_into( - tuple.output_mut(), - old_deserializers, - Some(tuple_id), - value, - old_deserializers.len(), - pool, - None, - )? - }; - len += 1; - } - - Ok(len) - }) + let mut iter = transaction.range(lower, Bound::Included(max), pool).await?; + let mut len = 0; + + while len < batch_size { + let Some((key, value)) = iter.try_next().await? else { + break; + }; + let tuple_id = TableCodec::decode_tuple_key(key, pk_ty)?; + let tuple = if len < batch.len() { + &mut batch[len] + } else { + batch.push(TupleImpl::from(Tuple::new( + None, + Vec::with_capacity(old_deserializers.len()), + ))); + batch + .last_mut() + .expect("batch contains the tuple that was just pushed") + }; + // SAFETY: without a scope every value is copied. + unsafe { + TableCodec::decode_tuple_into( + tuple.output_mut(), + old_deserializers, + Some(tuple_id), + value, + old_deserializers.len(), + pool, + None, + )? + }; + len += 1; + } + + Ok(len) + }) + .await } -pub(crate) fn visit_table_in_batches( +#[maybe_async] +pub(crate) async fn visit_table_in_batches( transaction: &T, table_codec: &mut TableCodec, table_id: TableId, @@ -122,7 +128,8 @@ where last_pk.as_ref(), &mut batch, REWRITE_BATCH_SIZE, - )?; + ) + .await?; if batch_len == 0 { break; } @@ -142,8 +149,9 @@ where Ok(()) } +#[maybe_async] #[allow(clippy::too_many_arguments)] -pub(crate) fn rewrite_table_in_batches( +pub(crate) async fn rewrite_table_in_batches( transaction: &mut T, table_codec: &mut TableCodec, table_id: TableId, @@ -156,7 +164,7 @@ pub(crate) fn rewrite_table_in_batches( where T: StorageTransaction, F: FnMut(&mut TupleImpl) -> Result<(), DatabaseError>, - G: FnMut(&mut T, &mut TableCodec, &TupleImpl) -> Result<(), DatabaseError>, + G: AsyncFnMut(&mut T, &mut TableCodec, &TupleImpl) -> Result<(), DatabaseError>, { let mut last_pk = None; let mut batch = Vec::with_capacity(REWRITE_BATCH_SIZE); @@ -171,7 +179,8 @@ where last_pk.as_ref(), &mut batch, REWRITE_BATCH_SIZE, - )?; + ) + .await?; if batch_len == 0 { break; } @@ -181,8 +190,10 @@ where for tuple in batch.iter_mut().take(batch_len) { rewrite(tuple)?; - transaction.append_tuple(table_codec, table_id, tuple, new_serializers, true)?; - after_write(transaction, table_codec, tuple)?; + transaction + .append_tuple(table_codec, table_id, tuple, new_serializers, true) + .await?; + after_write(transaction, table_codec, tuple).await?; } if batch_len < REWRITE_BATCH_SIZE { diff --git a/src/execution/ddl/truncate.rs b/src/execution/ddl/truncate.rs index f5ca9882..79e8a30e 100644 --- a/src/execution/ddl/truncate.rs +++ b/src/execution/ddl/truncate.rs @@ -34,8 +34,9 @@ impl<'a> WriteExecutor<'a> for Truncate<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Truncate<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -43,7 +44,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Truncate<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -51,7 +52,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Truncate<'a> { let TruncateOperator { table_id } = *self.op; let mut state = arena.local_state(plan_arena); let (transaction, table_codec) = state.transaction_codec_mut(); - transaction.drop_data(table_codec, table_id)?; + transaction.drop_data(table_codec, table_id).await?; arena.finish(); Ok(()) diff --git a/src/execution/dml/analyze.rs b/src/execution/dml/analyze.rs index 4f887f06..28ee4ee8 100644 --- a/src/execution/dml/analyze.rs +++ b/src/execution/dml/analyze.rs @@ -77,17 +77,18 @@ impl<'a> WriteExecutor<'a> for Analyze<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Analyze<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.done = false; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -140,7 +141,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Analyze<'a> { (columns, indexes) }; - while input.next_tuple(arena, plan_arena)? { + while input.next_tuple(arena, plan_arena).await? { let tuple = arena.take_tuple(); for (_, position, builder) in &mut columns { builder.append(tuple.value_at(*position)); @@ -167,7 +168,8 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Analyze<'a> { transaction, table_codec, plan_arena, - )?; + ) + .await?; arena.produce(crate::types::tuple::Tuple::new(None, values).into()); Ok(()) @@ -181,9 +183,10 @@ struct State { histogram_buckets: Option, } +#[maybe_async] impl<'a> Analyze<'a> { #[allow(clippy::too_many_arguments)] - fn persist_statistics( + async fn persist_statistics( table_id: TableId, table_name: &str, columns: Vec<(ColumnId, usize, HistogramMetaBuilder)>, @@ -196,13 +199,9 @@ impl<'a> Analyze<'a> { let mut values = Vec::with_capacity(builders.len()); for (column_id, _, builder) in columns { let meta = builder.build(); - transaction.save_column_statistics( - table_codec, - table_id, - column_id, - &meta, - plan_arena, - )?; + transaction + .save_column_statistics(table_codec, table_id, column_id, &meta, plan_arena) + .await?; applies.push(DDLApply::UpsertColumnStatistics { table_id, column_id, @@ -250,6 +249,7 @@ impl fmt::Display for AnalyzeOperator { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use crate::db::{CatalogKind, DataBaseBuilder, Database}; use crate::errors::DatabaseError; @@ -259,24 +259,25 @@ mod test { use crate::optimizer::core::cm_sketch::COUNT_MIN_SKETCH_STORAGE_PAGE_LEN; use crate::optimizer::core::histogram::{HistogramMetaBuilder, DEFAULT_HLL_PRECISION}; use crate::optimizer::core::statistics_meta::StatisticMetaLoader; - use crate::storage::{table_codec::TableCodec, InnerIter, Storage, StorageTransaction}; + use crate::storage::table_codec::TableCodec; + use crate::storage::{InnerIter, Storage, StorageTransaction}; use crate::types::value::DataValue; use std::ops::Bound; use tempfile::TempDir; #[test] - fn test_analyze() -> Result<(), DatabaseError> { - test_column_and_index_statistics_roundtrip()?; - test_meta_loader_uses_cache()?; - test_meta_loader_negative_cache()?; - test_clean_expired_index()?; - test_hll_precision_is_configurable()?; - test_statistics_relative_error_is_configurable()?; + async fn test_analyze() -> Result<(), DatabaseError> { + test_column_and_index_statistics_roundtrip().await?; + test_meta_loader_uses_cache().await?; + test_meta_loader_negative_cache().await?; + test_clean_expired_index().await?; + test_hll_precision_is_configurable().await?; + test_statistics_relative_error_is_configurable().await?; Ok(()) } - fn test_hll_precision_is_configurable() -> Result<(), DatabaseError> { + async fn test_hll_precision_is_configurable() -> Result<(), DatabaseError> { let (rows, distinct) = (1000, 500); let ndv_with = |precision: u8| -> Result { let mut builder = HistogramMetaBuilder::new(precision)?; @@ -296,13 +297,17 @@ mod test { (DataBaseBuilder::path("."), default), (DataBaseBuilder::path(".").hll_precision(4), coarse), ] { - let mut kite_sql = builder.build_in_memory()?; - kite_sql.ddl("create table t1 (id int primary key, a int)")?; - kite_sql.ddl("create index idx_a on t1 (a)")?; + let mut kite_sql = builder.build_in_memory().await?; + kite_sql + .ddl("create table t1 (id int primary key, a int)") + .await?; + kite_sql.ddl("create index idx_a on t1 (a)").await?; kite_sql - .run(format!("insert into t1 values {values}"))? - .done()?; - kite_sql.analyze("t1")?; + .run(format!("insert into t1 values {values}")) + .await? + .done() + .await?; + kite_sql.analyze("t1").await?; let table_name = "t1".to_string(); let table_id = kite_sql @@ -327,7 +332,7 @@ mod test { Ok(()) } - fn test_statistics_relative_error_is_configurable() -> Result<(), DatabaseError> { + async fn test_statistics_relative_error_is_configurable() -> Result<(), DatabaseError> { // Count-Min width is next_power_of_two(2 / error): 0.001 -> 2048, 0.01 -> 256, // and error_bound(n) = ceil(2n / width). for (builder, width) in [ @@ -337,16 +342,18 @@ mod test { 256, ), ] { - let mut kite_sql = builder.build_in_memory()?; - kite_sql.ddl("create table t1 (a int primary key)")?; + let mut kite_sql = builder.build_in_memory().await?; + kite_sql.ddl("create table t1 (a int primary key)").await?; // ANALYZE needs at least DEFAULT_NUM_OF_BUCKETS rows. let values = (0..DEFAULT_NUM_OF_BUCKETS) .map(|i| format!("({i})")) .join(", "); kite_sql - .run(format!("insert into t1 values {values}"))? - .done()?; - kite_sql.analyze("t1")?; + .run(format!("insert into t1 values {values}")) + .await? + .done() + .await?; + kite_sql.analyze("t1").await?; let table_id = kite_sql .state @@ -361,41 +368,45 @@ mod test { Ok(()) } - fn create_table( + async fn create_table( database: &mut Database, sql: &str, ) -> Result<(), DatabaseError> { - database.ddl(sql) + database.ddl(sql).await } - fn create_index( + async fn create_index( database: &mut Database, sql: &str, ) -> Result<(), DatabaseError> { - database.ddl(sql) + database.ddl(sql).await } - fn test_column_and_index_statistics_roundtrip() -> Result<(), DatabaseError> { + async fn test_column_and_index_statistics_roundtrip() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); let buckets = 10; + let storage = crate::storage::new_test_storage().await; let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) .histogram_buckets(buckets) - .build_rocksdb()?; + .build_with_storage(storage.clone())?; create_table( &mut kite_sql, "create table t1 (a int primary key, b int, c varchar)", - )?; - create_index(&mut kite_sql, "create index b_index on t1 (b)")?; - create_index(&mut kite_sql, "create index p_index on t1 (a, b)")?; + ) + .await?; + create_index(&mut kite_sql, "create index b_index on t1 (b)").await?; + create_index(&mut kite_sql, "create index p_index on t1 (a, b)").await?; // Keep this fixture independent of the default histogram bucket count. for i in 0..101 { kite_sql - .run(format!("insert into t1 values({i}, {}, NULL)", i % 20))? - .done()?; + .run(format!("insert into t1 values({i}, {}, NULL)", i % 20)) + .await? + .done() + .await?; } - kite_sql.analyze("t1")?; + kite_sql.analyze("t1").await?; let table_name = "t1".to_string(); let table_id = kite_sql @@ -447,8 +458,11 @@ mod test { // Reload the stored parts instead of reusing ANALYZE's in-memory cache. drop(kite_sql); - let mut reopened = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; - reopened.load(CatalogKind::Table(table_name.clone()))?; + let mut reopened = + DataBaseBuilder::path(temp_dir.path()).build_with_storage(storage.clone())?; + reopened + .load(CatalogKind::Table(table_name.clone())) + .await?; let loader = StatisticMetaLoader::new(reopened.state.meta_cache()); for index in 0..=2 { let meta = loader.load(table_id, index)?.unwrap(); @@ -483,7 +497,7 @@ mod test { loader.load_column(table_id, c_id).unwrap().null_count(), 101 ); - reopened.ddl("drop index t1.b_index")?; + reopened.ddl("drop index t1.b_index").await?; let loader = StatisticMetaLoader::new(reopened.state.meta_cache()); assert!(loader.load(table_id, 1)?.is_none()); assert_eq!( @@ -500,7 +514,8 @@ mod test { 0, 1, reopened.state.table_arena().borrow_mut() - )? + ) + .await? .is_none()); assert_eq!( tx.column_statistics( @@ -508,13 +523,14 @@ mod test { 0, b_id, reopened.state.table_arena().borrow_mut() - )? + ) + .await? .unwrap() .distinct_values_len(), 20 ); drop(tx); - reopened.ddl("alter table t1 drop column c")?; + reopened.ddl("alter table t1 drop column c").await?; let tx = reopened.storage.transaction()?; assert!(tx .column_statistics( @@ -522,7 +538,8 @@ mod test { 0, c_id, reopened.state.table_arena().borrow_mut() - )? + ) + .await? .is_none()); assert!(tx .column_statistics( @@ -530,16 +547,20 @@ mod test { 0, a_id, reopened.state.table_arena().borrow_mut() - )? + ) + .await? .is_some()); drop(tx); - let mut iter = reopened.run("select a from t1 where b=7 order by a")?; + let mut iter = reopened + .run("select a from t1 where b=7 order by a") + .await?; let mut rows = Vec::new(); while iter - .next_tuple(|_, row| rows.push(row[0].clone()))? + .next_tuple(|_, row| rows.push(row[0].clone())) + .await? .is_some() {} - iter.done()?; + iter.done().await?; assert_eq!(rows, [7, 27, 47, 67, 87].map(DataValue::Int32)); // A missing root must not publish partially reconstructed statistics. @@ -552,29 +573,36 @@ mod test { reopened.state.table_arena().borrow(), |key, _| tx.remove(key), )?; - tx.commit()?; + tx.commit().await?; drop(reopened); - let mut reopened = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; - assert!(matches!(reopened.load(CatalogKind::Table(table_name)), - Err(DatabaseError::InvalidValue(message)) if message == "statistics meta is incomplete")); + let mut reopened = + DataBaseBuilder::path(temp_dir.path()).build_with_storage(storage.clone())?; + assert!( + matches!(reopened.load(CatalogKind::Table(table_name)).await, + Err(DatabaseError::InvalidValue(message)) if message == "statistics meta is incomplete") + ); assert!(reopened.state.meta_cache().indexes.is_empty()); assert!(reopened.state.meta_cache().columns.is_empty()); Ok(()) } - fn test_meta_loader_uses_cache() -> Result<(), DatabaseError> { + async fn test_meta_loader_uses_cache() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - create_table(&mut kite_sql, "create table t1 (a int primary key, b int)")?; - create_index(&mut kite_sql, "create index b_index on t1 (b)")?; + create_table(&mut kite_sql, "create table t1 (a int primary key, b int)").await?; + create_index(&mut kite_sql, "create index b_index on t1 (b)").await?; for i in 0..DEFAULT_NUM_OF_BUCKETS + 1 { kite_sql - .run(format!("insert into t1 values({i}, {i})"))? - .done()?; + .run(format!("insert into t1 values({i}, {i})")) + .await? + .done() + .await?; } - kite_sql.analyze("t1")?; + kite_sql.analyze("t1").await?; let table_id = kite_sql .state @@ -597,16 +625,18 @@ mod test { ); let mut transaction = kite_sql.storage.transaction()?; let mut table_codec = TableCodec::default(); - let keys: Vec> = - table_codec.with_index_statistics_bound(0, 1, |min, max, pool| { - let mut iter = - transaction.range(Bound::Included(min), Bound::Included(max), pool)?; + let keys: Vec> = table_codec + .with_index_statistics_bound(0, 1, async |min, max, pool| { + let mut iter = transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await?; let mut keys = Vec::new(); - while let Some((key, _)) = iter.try_next()? { + while let Some((key, _)) = iter.try_next().await? { keys.push(key.to_vec()); } Ok(keys) - })?; + }) + .await?; for key in keys { transaction.remove(&key)?; } @@ -628,12 +658,14 @@ mod test { Ok(()) } - fn test_meta_loader_negative_cache() -> Result<(), DatabaseError> { + async fn test_meta_loader_negative_cache() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - create_table(&mut kite_sql, "create table t1 (a int primary key, b int)")?; - create_index(&mut kite_sql, "create index b_index on t1 (b)")?; + create_table(&mut kite_sql, "create table t1 (a int primary key, b int)").await?; + create_index(&mut kite_sql, "create index b_index on t1 (b)").await?; let table_id = kite_sql .state @@ -653,49 +685,60 @@ mod test { Ok(()) } - fn test_clean_expired_index() -> Result<(), DatabaseError> { + async fn test_clean_expired_index() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut kite_sql = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; + let mut kite_sql = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; - create_table(&mut kite_sql, "create table t1 (a int primary key, b int)")?; - create_index(&mut kite_sql, "create index b_index on t1 (b)")?; - create_index(&mut kite_sql, "create index p_index on t1 (a, b)")?; + create_table(&mut kite_sql, "create table t1 (a int primary key, b int)").await?; + create_index(&mut kite_sql, "create index b_index on t1 (b)").await?; + create_index(&mut kite_sql, "create index p_index on t1 (a, b)").await?; for i in 0..DEFAULT_NUM_OF_BUCKETS + 1 { kite_sql - .run(format!("insert into t1 values({i}, {i})"))? - .done()?; + .run(format!("insert into t1 values({i}, {i})")) + .await? + .done() + .await?; } - kite_sql.analyze("t1")?; + kite_sql.analyze("t1").await?; let count = { let transaction = kite_sql.storage.transaction()?; let mut table_codec = TableCodec::default(); - table_codec.with_statistics_bound(0, |min, max, pool| { - let mut iter = - transaction.range(Bound::Included(min), Bound::Included(max), pool)?; - let mut count = 0; - while iter.try_next()?.is_some() { - count += 1; - } - Ok(count) - })? + table_codec + .with_statistics_bound(0, async |min, max, pool| { + let mut iter = transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await?; + let mut count = 0; + while iter.try_next().await?.is_some() { + count += 1; + } + Ok(count) + }) + .await? }; assert!(count > 3); - kite_sql.ddl("alter table t1 drop column b")?; - kite_sql.analyze("t1")?; + kite_sql.ddl("alter table t1 drop column b").await?; + kite_sql.analyze("t1").await?; let transaction = kite_sql.storage.transaction()?; let mut table_codec = TableCodec::default(); - let keys = table_codec.with_statistics_bound(0, |min, max, pool| { - let mut iter = transaction.range(Bound::Included(min), Bound::Included(max), pool)?; - let mut keys = 0; - while iter.try_next()?.is_some() { - keys += 1; - } - Ok(keys) - })?; + let keys = table_codec + .with_statistics_bound(0, async |min, max, pool| { + let mut iter = transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await?; + let mut keys = 0; + while iter.try_next().await?.is_some() { + keys += 1; + } + Ok(keys) + }) + .await?; let table_id = kite_sql .state .names() @@ -713,17 +756,21 @@ mod test { + index_statistics.histogram().buckets_len(); assert_eq!(keys, expected_keys); drop(transaction); - kite_sql.ddl("drop table t1")?; + kite_sql.ddl("drop table t1").await?; assert!(kite_sql.state.meta_cache().columns.is_empty()); assert!(kite_sql.state.meta_cache().indexes.is_empty()); let transaction = kite_sql.storage.transaction()?; - table_codec.with_statistics_bound(0, |min, max, pool| { - assert!(transaction - .range(Bound::Included(min), Bound::Included(max), pool)? - .try_next()? - .is_none()); - Ok(()) - })?; + table_codec + .with_statistics_bound(0, async |min, max, pool| { + assert!(transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await? + .try_next() + .await? + .is_none()); + Ok(()) + }) + .await?; Ok(()) } } diff --git a/src/execution/dml/copy_from_file.rs b/src/execution/dml/copy_from_file.rs index 00e472a5..ab488c55 100644 --- a/src/execution/dml/copy_from_file.rs +++ b/src/execution/dml/copy_from_file.rs @@ -40,8 +40,9 @@ impl<'a> WriteExecutor<'a> for CopyFromFile<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CopyFromFile<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -49,7 +50,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CopyFromFile<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -100,7 +101,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CopyFromFile<'a> { tuple_builder.build_with_row(chunk.output_mut(), record.iter().take(column_count))?; let mut state = arena.local_state(plan_arena); let (transaction, table_codec) = state.transaction_codec_mut(); - transaction.append_tuple(table_codec, table.id(), &chunk, &serializers, false)?; + transaction + .append_tuple(table_codec, table.id(), &chunk, &serializers, false) + .await?; } arena.finish(); @@ -110,6 +113,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CopyFromFile<'a> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::binder::copy::ExtSource; @@ -123,16 +127,19 @@ mod tests { use tempfile::TempDir; #[test] - fn read_csv() -> Result<(), DatabaseError> { + async fn read_csv() -> Result<(), DatabaseError> { let csv = "1,1.5,one\n2,2.5,two\n"; let mut file = tempfile::NamedTempFile::new().expect("failed to create temp file"); write!(file, "{csv}").expect("failed to write file"); let tmp_dir = TempDir::new()?; - let mut db = DataBaseBuilder::path(tmp_dir.path()).build_rocksdb()?; - db.ddl("create table test_copy (a int primary key, b float, c varchar(10))")?; - db.load(CatalogKind::Table("test_copy".to_string()))?; + let mut db = DataBaseBuilder::path(tmp_dir.path()) + .build_in_memory() + .await?; + db.ddl("create table test_copy (a int primary key, b float, c varchar(10))") + .await?; + db.load(CatalogKind::Table("test_copy".to_string())).await?; fn test_column( name: &str, @@ -181,16 +188,17 @@ mod tests { &transaction, ); - assert!(executor.next_tuple()?.is_none()); + assert!(executor.next_tuple().await?.is_none()); drop(executor); - transaction.commit()?; + transaction.commit().await?; - let mut iter = db.run("select count(*) from test_copy")?; + let mut iter = db.run("select count(*) from test_copy").await?; let count = iter - .next_tuple(|_, tuple| tuple[0].to_string())? + .next_tuple(|_, tuple| tuple[0].to_string()) + .await? .expect("count row"); assert_eq!(count, "2"); - iter.done()?; + iter.done().await?; Ok(()) } diff --git a/src/execution/dml/copy_to_file.rs b/src/execution/dml/copy_to_file.rs index 6809fb9c..3e991f5c 100644 --- a/src/execution/dml/copy_to_file.rs +++ b/src/execution/dml/copy_to_file.rs @@ -79,17 +79,18 @@ impl CopyToFile<'_> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CopyToFile<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.done = false; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -100,7 +101,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CopyToFile<'a> { } let mut writer = self.create_writer()?; - while self.input.next_tuple(arena, plan_arena)? { + while self.input.next_tuple(arena, plan_arena).await? { writer.write_record(arena.result_tuple().iter().map(|v| v.to_string()))?; } writer.flush().map_err(DatabaseError::from)?; @@ -118,6 +119,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for CopyToFile<'a> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::binder::copy::ExtSource; @@ -129,7 +131,7 @@ mod tests { use tempfile::TempDir; #[test] - fn read_csv() -> Result<(), DatabaseError> { + async fn read_csv() -> Result<(), DatabaseError> { let tmp_dir = TempDir::new()?; let file_path = tmp_dir.path().join("test.csv"); @@ -146,12 +148,24 @@ mod tests { }; let temp_dir = TempDir::new().unwrap(); - let mut db = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; - db.ddl("create table t1 (a int primary key, b float, c varchar(10))")?; - db.load(CatalogKind::Table("t1".to_string()))?; - db.run("insert into t1 values (1, 1.1, 'foo')")?.done()?; - db.run("insert into t1 values (2, 2.0, 'fooo')")?.done()?; - db.run("insert into t1 values (3, 2.1, 'Kite')")?.done()?; + let mut db = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; + db.ddl("create table t1 (a int primary key, b float, c varchar(10))") + .await?; + db.load(CatalogKind::Table("t1".to_string())).await?; + db.run("insert into t1 values (1, 1.1, 'foo')") + .await? + .done() + .await?; + db.run("insert into t1 values (2, 2.0, 'fooo')") + .await? + .done() + .await?; + db.run("insert into t1 values (3, 2.1, 'Kite')") + .await? + .done() + .await?; let mut plan_arena = crate::planner::PlanArena::new(db.state.table_arena()); let transaction = db.storage.transaction()?; @@ -168,7 +182,10 @@ mod tests { &transaction, ); - let tuple = executor.next_tuple()?.expect("executor should yield once"); + let tuple = executor + .next_tuple() + .await? + .expect("executor should yield once"); let mut rdr = csv::Reader::from_path(file_path)?; let headers = rdr.headers()?.clone(); diff --git a/src/execution/dml/delete.rs b/src/execution/dml/delete.rs index 79d26fe5..0fdf9857 100644 --- a/src/execution/dml/delete.rs +++ b/src/execution/dml/delete.rs @@ -49,17 +49,18 @@ impl<'a> WriteExecutor<'a> for Delete<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Delete<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.done = false; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -98,7 +99,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Delete<'a> { }; let mut deleted_count = 0; - while input.next_tuple(arena, plan_arena)? { + while input.next_tuple(arena, plan_arena).await? { let tuple = arena.take_tuple(); let Some(tuple_id) = tuple.pk() else { continue; @@ -108,12 +109,17 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Delete<'a> { let mut state = arena.local_state(plan_arena); let (values, transaction, table_codec) = state.tuple_transaction_codec_mut(); let index = Index::new(*index_id, values, *index_ty); - transaction.del_index(table_codec, table_id, &index, tuple_id)?; + transaction + .del_index(table_codec, table_id, &index, tuple_id) + .await?; } let mut state = arena.local_state(plan_arena); + let (transaction, table_codec) = state.transaction_codec_mut(); - transaction.remove_tuple(table_codec, table_id, tuple_id)?; + transaction + .remove_tuple(table_codec, table_id, tuple_id) + .await?; deleted_count += 1; } diff --git a/src/execution/dml/insert.rs b/src/execution/dml/insert.rs index ad89a535..cac6bd0e 100644 --- a/src/execution/dml/insert.rs +++ b/src/execution/dml/insert.rs @@ -60,17 +60,18 @@ impl<'a> WriteExecutor<'a> for Insert<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Insert<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.done = false; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -91,7 +92,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Insert<'a> { } let mut inserted_count = 0; - while input.next_tuple(arena, plan_arena)? { + while input.next_tuple(arena, plan_arena).await? { tuple.clear(); arena.result_mut().drain_values(|value| tuple.push(value)); for (i, column) in table.columns.iter().enumerate() { @@ -114,17 +115,21 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Insert<'a> { let mut state = arena.local_state(plan_arena); let (values, transaction, table_codec) = state.tuple_transaction_codec_mut(); let index = Index::new(index_meta.id, values, index_meta.ty); - transaction.add_index(table_codec, table.table_id, index, tuple_id)?; + transaction + .add_index(table_codec, table.table_id, index, tuple_id) + .await?; } let mut state = arena.local_state(plan_arena); let (transaction, table_codec) = state.transaction_codec_mut(); - transaction.append_tuple( - table_codec, - table.table_id, - &*tuple, - &table.serializers, - *is_overwrite, - )?; + transaction + .append_tuple( + table_codec, + table.table_id, + &*tuple, + &table.serializers, + *is_overwrite, + ) + .await?; inserted_count += 1; } arena.produce(TupleBuilder::build_result(inserted_count.to_string()).into()); diff --git a/src/execution/dml/update.rs b/src/execution/dml/update.rs index a30d843b..6331845d 100644 --- a/src/execution/dml/update.rs +++ b/src/execution/dml/update.rs @@ -52,17 +52,18 @@ impl<'a> WriteExecutor<'a> for Update<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Update<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.done = false; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -80,7 +81,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Update<'a> { let mut updated_count = 0; let mut values = Vec::with_capacity(value_exprs.len()); - while input.next_tuple(arena, plan_arena)? { + while input.next_tuple(arena, plan_arena).await? { let mut is_overwrite = true; let mut tuple = arena.take_tuple(); @@ -94,7 +95,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Update<'a> { let mut state = arena.local_state(plan_arena); let (values, transaction, table_codec) = state.tuple_transaction_codec_mut(); let old_index = Index::new(index_meta.id, values, index_meta.ty); - transaction.del_index(table_codec, table.table_id, &old_index, &old_pk)?; + transaction + .del_index(table_codec, table.table_id, &old_index, &old_pk) + .await?; } // Every expression reads the old row, so evaluate them all before writing. for (_, expr) in value_exprs.iter() { @@ -112,7 +115,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Update<'a> { if primary_key_changed { let mut state = arena.local_state(plan_arena); let (transaction, table_codec) = state.transaction_codec_mut(); - transaction.remove_tuple(table_codec, table.table_id, &old_pk)?; + transaction + .remove_tuple(table_codec, table.table_id, &old_pk) + .await?; is_overwrite = false; } @@ -122,22 +127,28 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Update<'a> { let mut state = arena.local_state(plan_arena); let (values, transaction, table_codec) = state.tuple_transaction_codec_mut(); let new_index = Index::new(index_meta.id, values, index_meta.ty); - transaction.add_index(table_codec, table.table_id, new_index, &new_pk)?; + transaction + .add_index(table_codec, table.table_id, new_index, &new_pk) + .await?; } tuple.set_pk(Some(new_pk)); let mut state = arena.local_state(plan_arena); let (transaction, table_codec) = state.transaction_codec_mut(); let stamp = if is_overwrite { 0 } else { table_codec.stamp() }; - table_codec.with_stamp(stamp, |table_codec| { - transaction.append_tuple( - table_codec, - table.table_id, - &tuple, - &table.serializers, - is_overwrite, - ) - })?; + table_codec + .with_stamp(stamp, async |table_codec| { + transaction + .append_tuple( + table_codec, + table.table_id, + &tuple, + &table.serializers, + is_overwrite, + ) + .await + }) + .await?; updated_count += 1; } arena.produce(TupleBuilder::build_result(updated_count.to_string()).into()); diff --git a/src/execution/dql/aggregate/avg.rs b/src/execution/dql/aggregate/avg.rs index 8308b773..d7bbd096 100644 --- a/src/execution/dql/aggregate/avg.rs +++ b/src/execution/dql/aggregate/avg.rs @@ -103,6 +103,7 @@ impl Accumulator for AvgAccumulator { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; diff --git a/src/execution/dql/aggregate/count.rs b/src/execution/dql/aggregate/count.rs index 97d794d9..5690a680 100644 --- a/src/execution/dql/aggregate/count.rs +++ b/src/execution/dql/aggregate/count.rs @@ -96,6 +96,7 @@ impl Accumulator for DistinctCountAccumulator { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; diff --git a/src/execution/dql/aggregate/hash_agg.rs b/src/execution/dql/aggregate/hash_agg.rs index 9d0e4700..bdb9828f 100644 --- a/src/execution/dql/aggregate/hash_agg.rs +++ b/src/execution/dql/aggregate/hash_agg.rs @@ -73,8 +73,9 @@ impl<'a> ReadExecutor<'a> for HashAggExecutor<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for HashAggExecutor<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -83,10 +84,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for HashAggExecutor<'a> self.accs.clear(); self.groups.clear(); self.output = None; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -97,7 +98,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for HashAggExecutor<'a> let hasher = FxBuildHasher::default(); let groups = &mut self.groups; - while self.input.next_tuple(arena, plan_arena)? { + while self.input.next_tuple(arena, plan_arena).await? { let tuple = arena.result_tuple(); // Evaluate the key in place, and drop it again if its group exists. let start = self.keys.len(); @@ -159,6 +160,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for HashAggExecutor<'a> // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::errors::DatabaseError; @@ -176,18 +178,15 @@ mod test { use crate::planner::operator::Operator; use crate::planner::test::PlanArenaTestExt; use crate::planner::{Childrens, LogicalPlan}; - use crate::storage::rocksdb::RocksStorage; use crate::storage::Storage; use crate::types::value::DataValue; use crate::types::LogicalType; - use tempfile::TempDir; #[test] - fn test_hash_agg() -> Result<(), DatabaseError> { + async fn test_hash_agg() -> Result<(), DatabaseError> { let table_cache = crate::storage::TableCache::default(); - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path()).unwrap(); + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let desc = ColumnDesc::new(LogicalType::Integer, None, false, None)?; let table_arena = crate::planner::TableArenaCell::default(); @@ -266,7 +265,8 @@ mod test { &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; assert_eq!(tuples.len(), 2); diff --git a/src/execution/dql/aggregate/min_max.rs b/src/execution/dql/aggregate/min_max.rs index c1aacf8c..ebd5f5a0 100644 --- a/src/execution/dql/aggregate/min_max.rs +++ b/src/execution/dql/aggregate/min_max.rs @@ -77,6 +77,7 @@ impl Accumulator for MinMaxAccumulator { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; diff --git a/src/execution/dql/aggregate/mod.rs b/src/execution/dql/aggregate/mod.rs index eeaa01b7..92f8513c 100644 --- a/src/execution/dql/aggregate/mod.rs +++ b/src/execution/dql/aggregate/mod.rs @@ -32,6 +32,7 @@ use crate::execution::dql::aggregate::sum::{DistinctSumAccumulator, SumAccumulat use crate::execution::dql::aggregate::total::TotalAccumulator; use crate::expression::agg::AggKind; use crate::expression::ScalarExpression; +use crate::maybe_async::MaybeSend; use crate::planner::ExprRef; use crate::planner::MetaArena; use crate::types::tuple::{Tuple, TupleLike}; @@ -41,7 +42,7 @@ use std::borrow::Cow; /// Tips: Idea for sqlrs /// An accumulator represents a stateful object that lives throughout the evaluation of multiple /// rows and generically accumulates values. -pub trait Accumulator { +pub trait Accumulator: MaybeSend { /// updates the accumulator's state from a vector of arrays. fn update_value(&mut self, value: &DataValue) -> Result<(), DatabaseError>; diff --git a/src/execution/dql/aggregate/simple_agg.rs b/src/execution/dql/aggregate/simple_agg.rs index a41b0924..292cb1f8 100644 --- a/src/execution/dql/aggregate/simple_agg.rs +++ b/src/execution/dql/aggregate/simple_agg.rs @@ -49,17 +49,18 @@ impl<'a> ReadExecutor<'a> for SimpleAggExecutor<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SimpleAggExecutor<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.returned = false; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -70,7 +71,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SimpleAggExecutor<' } prepare_accumulators(&mut self.accs, self.agg_calls, plan_arena)?; - while self.input.next_tuple(arena, plan_arena)? { + while self.input.next_tuple(arena, plan_arena).await? { let tuple = arena.result_tuple(); update_accumulators(&mut self.accs, self.agg_calls, tuple, plan_arena)?; } diff --git a/src/execution/dql/aggregate/stream_agg.rs b/src/execution/dql/aggregate/stream_agg.rs index 0f36eee9..46c5d608 100644 --- a/src/execution/dql/aggregate/stream_agg.rs +++ b/src/execution/dql/aggregate/stream_agg.rs @@ -70,8 +70,9 @@ impl<'a> ReadExecutor<'a> for StreamAggExecutor<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for StreamAggExecutor<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -80,10 +81,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for StreamAggExecutor<' self.in_group = false; self.carried = false; self.carried_args.clear(); - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -97,7 +98,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for StreamAggExecutor<' arena.bytes_pool().recycle(&mut self.carried_args); } loop { - if !self.input.next_tuple(arena, plan_arena)? { + if !self.input.next_tuple(arena, plan_arena).await? { if !mem::take(&mut self.in_group) { arena.finish(); return Ok(()); @@ -150,6 +151,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for StreamAggExecutor<' // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::StreamAggExecutor; use crate::catalog::{ColumnCatalog, ColumnDesc}; @@ -162,13 +164,12 @@ mod tests { use crate::planner::operator::Operator; use crate::planner::test::PlanArenaTestExt; use crate::planner::{Childrens, LogicalPlan}; - use crate::storage::memory::MemoryStorage; use crate::storage::Storage; use crate::types::value::DataValue; use crate::types::LogicalType; #[test] - fn aggregates_sorted_groups() -> Result<(), DatabaseError> { + async fn aggregates_sorted_groups() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let desc = ColumnDesc::new(LogicalType::Integer, None, false, None)?; @@ -212,7 +213,7 @@ mod tests { force_spill: false, }; let table_cache = crate::storage::TableCache::default(); - let storage = MemoryStorage::new(); + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let rows = try_collect(execute_input::<_, StreamAggExecutor>( @@ -220,7 +221,8 @@ mod tests { &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; assert_eq!( rows.into_iter() @@ -235,7 +237,7 @@ mod tests { } #[test] - fn empty_input_returns_no_groups() -> Result<(), DatabaseError> { + async fn empty_input_returns_no_groups() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let column = plan_arena.alloc_column(ColumnCatalog::new( @@ -256,7 +258,7 @@ mod tests { force_spill: false, }; let table_cache = crate::storage::TableCache::default(); - let storage = MemoryStorage::new(); + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let rows = try_collect(execute_input::<_, StreamAggExecutor>( @@ -264,7 +266,8 @@ mod tests { &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; assert!(rows.is_empty()); Ok(()) diff --git a/src/execution/dql/aggregate/stream_distinct.rs b/src/execution/dql/aggregate/stream_distinct.rs index 6d525659..e31e44fc 100644 --- a/src/execution/dql/aggregate/stream_distinct.rs +++ b/src/execution/dql/aggregate/stream_distinct.rs @@ -47,23 +47,24 @@ impl<'a> ReadExecutor<'a> for StreamDistinctExecutor<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for StreamDistinctExecutor<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.last_keys = None; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { - if !self.input.next_tuple(arena, plan_arena)? { + if !self.input.next_tuple(arena, plan_arena).await? { if let Some(last_keys) = self.last_keys.take() { drop(std::mem::replace(arena.result_mut(), last_keys)); arena.resume(); @@ -89,6 +90,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for StreamDistinctExecu // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::errors::DatabaseError; @@ -104,18 +106,18 @@ mod tests { use crate::planner::operator::Operator; use crate::planner::test::PlanArenaTestExt; use crate::planner::{Childrens, LogicalPlan}; - use crate::storage::rocksdb::RocksStorage; + use crate::storage::TestStorage; use crate::storage::{Storage, TableCache}; use crate::types::value::DataValue; use crate::types::LogicalType; use tempfile::TempDir; #[allow(clippy::type_complexity)] - fn build_test_storage() -> Result<(TableCache, TempDir, RocksStorage), DatabaseError> { + async fn build_test_storage() -> Result<(TableCache, TempDir, TestStorage), DatabaseError> { let table_cache = crate::storage::TableCache::default(); let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + let storage = crate::storage::new_test_storage().await; Ok((table_cache, temp_dir, storage)) } @@ -136,7 +138,7 @@ mod tests { } #[test] - fn stream_distinct_single_column_sorted() -> Result<(), DatabaseError> { + async fn stream_distinct_single_column_sorted() -> Result<(), DatabaseError> { let desc = ColumnDesc::new(LogicalType::Integer, None, false, None)?; let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); @@ -171,14 +173,15 @@ mod tests { unreachable!() }; - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let tuples = try_collect(execute_input::<_, StreamDistinctExecutor>( (agg, plan.childrens.iter().next().unwrap()), &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; let actual = tuples .into_iter() @@ -191,7 +194,7 @@ mod tests { } #[test] - fn stream_distinct_multi_column_sorted() -> Result<(), DatabaseError> { + async fn stream_distinct_multi_column_sorted() -> Result<(), DatabaseError> { let desc = ColumnDesc::new(LogicalType::Integer, None, false, None)?; let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); @@ -229,14 +232,15 @@ mod tests { unreachable!() }; - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let tuples = try_collect(execute_input::<_, StreamDistinctExecutor>( (agg, plan.childrens.iter().next().unwrap()), &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; let actual = tuples .into_iter() diff --git a/src/execution/dql/aggregate/sum.rs b/src/execution/dql/aggregate/sum.rs index 6c3aeefe..00df3378 100644 --- a/src/execution/dql/aggregate/sum.rs +++ b/src/execution/dql/aggregate/sum.rs @@ -149,6 +149,7 @@ impl Accumulator for DistinctSumAccumulator { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; diff --git a/src/execution/dql/describe.rs b/src/execution/dql/describe.rs index 4b636b51..95deef12 100644 --- a/src/execution/dql/describe.rs +++ b/src/execution/dql/describe.rs @@ -69,8 +69,9 @@ impl<'a> ReadExecutor<'a> for Describe { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Describe { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -80,7 +81,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Describe { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, diff --git a/src/execution/dql/dummy.rs b/src/execution/dql/dummy.rs index c3ad1d94..7044260e 100644 --- a/src/execution/dql/dummy.rs +++ b/src/execution/dql/dummy.rs @@ -43,8 +43,9 @@ impl<'a> ReadExecutor<'a> for Dummy { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Dummy { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -53,7 +54,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Dummy { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, _: &mut ExecMetaArena, diff --git a/src/execution/dql/explain.rs b/src/execution/dql/explain.rs index 68fa0ff7..7747aebd 100644 --- a/src/execution/dql/explain.rs +++ b/src/execution/dql/explain.rs @@ -40,8 +40,9 @@ impl<'a> ReadExecutor<'a> for Explain<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Explain<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -50,7 +51,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Explain<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, diff --git a/src/execution/dql/external_sort.rs b/src/execution/dql/external_sort.rs index ffc2de11..3b49227c 100644 --- a/src/execution/dql/external_sort.rs +++ b/src/execution/dql/external_sort.rs @@ -66,17 +66,18 @@ impl<'a> ReadExecutor<'a> for ExternalSort<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ExternalSort<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.rows = None; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -100,7 +101,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ExternalSort<'a> { let sort_fields = self.sort_fields; let mut rows = SpillVec::new().on_flush(move |_, rows| sort_segment(sort_fields, rows)); let mut runs = Vec::new(); - while self.input.next_tuple(arena, plan_arena)? { + while self.input.next_tuple(arena, plan_arena).await? { let tuple = arena.take_tuple(); let row = SortRow::new(sort_fields, tuple, plan_arena)?; if let Some(segment) = rows.push(row)? { @@ -288,6 +289,7 @@ fn merge_pass<'on_flush>( // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod test { use super::{finish_sort, sort_segment, Run, MERGE_FAN_IN}; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/execution/dql/filter.rs b/src/execution/dql/filter.rs index aa4e1626..7f6e6fef 100644 --- a/src/execution/dql/filter.rs +++ b/src/execution/dql/filter.rs @@ -42,22 +42,23 @@ impl<'a> ReadExecutor<'a> for Filter<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Filter<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { loop { - if !self.input.next_tuple(arena, plan_arena)? { + if !self.input.next_tuple(arena, plan_arena).await? { arena.finish(); return Ok(()); }; diff --git a/src/execution/dql/function_scan.rs b/src/execution/dql/function_scan.rs index 9a4913f3..d557cba7 100644 --- a/src/execution/dql/function_scan.rs +++ b/src/execution/dql/function_scan.rs @@ -14,15 +14,14 @@ use crate::errors::DatabaseError; use crate::execution::{ExecBuilder, ExecContext, ExecNode, ExecutorNode, ReadExecutor}; -use crate::expression::function::table::TableFunction; +use crate::expression::function::table::{TableFunction, TableFunctionIter}; use crate::planner::operator::function_scan::FunctionScanOperator; use crate::planner::{ExecMetaArena, MetaArena}; use crate::storage::StorageTransaction; -use crate::types::tuple::Tuple; pub struct FunctionScan<'a> { table_function: &'a TableFunction, - iter: Option>>>, + iter: Option, } impl<'a> From<&'a FunctionScanOperator> for FunctionScan<'a> { @@ -47,8 +46,9 @@ impl<'a> ReadExecutor<'a> for FunctionScan<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for FunctionScan<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -57,7 +57,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for FunctionScan<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, diff --git a/src/execution/dql/index_scan.rs b/src/execution/dql/index_scan.rs index 95e6d85b..6ac57203 100644 --- a/src/execution/dql/index_scan.rs +++ b/src/execution/dql/index_scan.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::errors::DatabaseError; -use crate::execution::dql::seq_scan::next_scan_tuple; +use crate::execution::dql::seq_scan::scan_predicate_holds; use crate::execution::{ ExecBuilder, ExecContext, ExecNode, ExecutorNode, IterSlot, ReadExecutor, StorageIter, }; @@ -132,8 +132,9 @@ impl<'a> ReadExecutor<'a> for IndexScan<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexScan<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -149,7 +150,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexScan<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -185,13 +186,18 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexScan<'a> { let iter = state.iters[self.iter].as_index(); let bound_buffers = state.bound_buffers; let output = state.result.output_mut(); - if next_scan_tuple( - self.op, - |table_codec, tuple| iter.next_tuple_into(view, bound_buffers, table_codec, tuple), - state.table_codec, - output, - plan_arena, - )? { + let found = loop { + if !iter + .next_tuple_into(view, bound_buffers, state.table_codec, output) + .await? + { + break false; + } + if scan_predicate_holds(self.op, output, plan_arena)? { + break true; + } + }; + if found { arena.resume(); } else { arena.finish(); @@ -202,12 +208,12 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexScan<'a> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::db::test::build_table; use crate::execution::ExecStatus; use crate::planner::{PlanArena, TableArenaCell}; - use crate::storage::memory::MemoryStorage; use crate::storage::table_codec::TableCodec; use crate::storage::{Storage, TableCache, TupleDeserializers}; use crate::types::index::RuntimeIndexProbe; @@ -217,14 +223,15 @@ mod tests { use std::ops::Bound; #[test] - fn reset_rebinds_probe_after_partial_read_eof_and_missing_key() -> Result<(), DatabaseError> { - let storage = MemoryStorage::new(); + async fn reset_rebinds_probe_after_partial_read_eof_and_missing_key( + ) -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let mut tx = storage.transaction()?; let mut names = crate::storage::NameMapping::default(); let mut tables = TableCache::default(); let cell = TableArenaCell::default(); let mut meta = PlanArena::new(&cell); - build_table(&mut names, &mut tables, &mut tx, &mut meta)?; + build_table(&mut names, &mut tables, &mut tx, &mut meta).await?; let meta = PlanArena::new(&cell); let serializers = [ LogicalType::Integer.serializable(), @@ -247,7 +254,8 @@ mod tests { .into(), &serializers, false, - )?; + ) + .await?; } let table = names.table(&tables, "t1").unwrap(); let op = TableScanOperator { @@ -278,7 +286,7 @@ mod tests { min: Bound::Included(DataValue::Int32(1)), max: Bound::Included(DataValue::Int32(4)), }); - scan.next_tuple(&mut arena, &mut meta)?; + scan.next_tuple(&mut arena, &mut meta).await?; assert_eq!(arena.result_tuple()[0], DataValue::Int32(1)); let iterator = &arena.arena.iters[slot] as *const _; for (probe, expected) in [ @@ -301,15 +309,15 @@ mod tests { (RuntimeIndexProbe::Eq(DataValue::Int32(1)), vec![1]), ] { arena.push_runtime_probe(probe); - scan.reset(&mut arena, &mut meta)?; + scan.reset(&mut arena, &mut meta).await?; // An initialized scan is ready now, not at its next pull. assert!(arena.buffer.runtime_probe_stack.is_empty()); for id in expected { - scan.next_tuple(&mut arena, &mut meta)?; + scan.next_tuple(&mut arena, &mut meta).await?; assert_eq!(arena.result_tuple()[0], DataValue::Int32(id)); assert_eq!(&arena.arena.iters[slot] as *const _, iterator); } - scan.next_tuple(&mut arena, &mut meta)?; + scan.next_tuple(&mut arena, &mut meta).await?; assert_eq!(arena.buffer.result.status, Some(ExecStatus::End)); assert!(arena.buffer.runtime_probe_stack.is_empty()); } diff --git a/src/execution/dql/join/hash_join.rs b/src/execution/dql/join/hash_join.rs index 4dc3c28b..5ff80ae1 100644 --- a/src/execution/dql/join/hash_join.rs +++ b/src/execution/dql/join/hash_join.rs @@ -131,6 +131,7 @@ impl<'a> HashJoin<'a> { } } +#[maybe_async] impl<'a> HashJoin<'a> { fn eval_keys( on_keys: &[ExprRef], @@ -147,7 +148,7 @@ impl<'a> HashJoin<'a> { Ok(()) } - fn initialize_build( + async fn initialize_build( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -165,7 +166,7 @@ impl<'a> HashJoin<'a> { let mut rows: Vec = Vec::new(); let len = self.on_left_keys.len(); - while self.left_input.next_tuple(arena, plan_arena)? { + while self.left_input.next_tuple(arena, plan_arena).await? { let tuple = arena.take_tuple(); let row = rows.len(); // Evaluate the key in place, and drop it again if the key is already there. @@ -260,18 +261,19 @@ impl<'a> ReadExecutor<'a> for HashJoin<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for HashJoin<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.state = HashJoinState::Build; - self.left_input.reset(arena, plan_arena)?; - self.right_input.reset(arena, plan_arena) + self.left_input.reset(arena, plan_arena).await?; + self.right_input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -280,7 +282,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for HashJoin<'a> { return Err(err); } - self.initialize_build(arena, plan_arena)?; + self.initialize_build(arena, plan_arena).await?; let mut state = std::mem::replace(&mut self.state, HashJoinState::End); loop { @@ -294,7 +296,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for HashJoin<'a> { } => { let probe_finished = loop { if probe_state.is_none() { - if !self.right_input.next_tuple(arena, plan_arena)? { + if !self.right_input.next_tuple(arena, plan_arena).await? { break true; } let tuple = arena.take_tuple(); @@ -386,6 +388,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for HashJoin<'a> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::errors::DatabaseError; @@ -401,11 +404,9 @@ mod test { use crate::planner::operator::Operator; use crate::planner::test::PlanArenaTestExt; use crate::planner::{Childrens, ExprRef, LogicalPlan}; - use crate::storage::rocksdb::RocksStorage; use crate::storage::Storage; use crate::types::value::DataValue; use crate::types::LogicalType; - use tempfile::TempDir; fn optimize_exprs( plan: LogicalPlan, @@ -502,9 +503,8 @@ mod test { } #[test] - fn test_inner_join() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_inner_join() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -540,7 +540,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; assert_eq!(tuples.len(), 3); @@ -561,9 +561,8 @@ mod test { } #[test] - fn test_left_join() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_left_join() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -599,7 +598,8 @@ mod test { &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; assert_eq!(tuples.len(), 4); @@ -625,9 +625,8 @@ mod test { } #[test] - fn test_right_join() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_right_join() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -663,7 +662,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; assert_eq!(tuples.len(), 4); @@ -688,9 +687,8 @@ mod test { } #[test] - fn test_right_join_filter_only_left_columns() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_right_join_filter_only_left_columns() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -771,7 +769,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; assert_eq!(tuples.len(), 1); assert_eq!( @@ -787,9 +785,8 @@ mod test { } #[test] - fn test_full_join() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_full_join() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -825,7 +822,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; assert_eq!(tuples.len(), 5); diff --git a/src/execution/dql/join/index_probe_join.rs b/src/execution/dql/join/index_probe_join.rs index 7a07cd75..ea5a645e 100644 --- a/src/execution/dql/join/index_probe_join.rs +++ b/src/execution/dql/join/index_probe_join.rs @@ -47,21 +47,22 @@ impl<'a> ReadExecutor<'a> for IndexProbeJoin<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexProbeJoin<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { - self.left_input.reset(arena, plan_arena) + self.left_input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { - while self.left_input.next_tuple(arena, plan_arena)? { + while self.left_input.next_tuple(arena, plan_arena).await? { let mut left = arena.take_tuple(); let key = plan_arena .expression(self.op.probe) @@ -74,8 +75,8 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexProbeJoin<'a> continue; } arena.push_runtime_probe(RuntimeIndexProbe::Eq(key)); - self.right_input.reset(arena, plan_arena)?; - if self.right_input.next_tuple(arena, plan_arena)? + self.right_input.reset(arena, plan_arena).await?; + if self.right_input.next_tuple(arena, plan_arena).await? && predicates_matched(&self.op.predicates, &left, arena.result_tuple(), plan_arena)? { let mut right = arena.take_tuple(); @@ -93,6 +94,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for IndexProbeJoin<'a> // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; @@ -102,7 +104,7 @@ mod tests { use crate::planner::operator::Operator; use crate::planner::test::PlanArenaTestExt; use crate::planner::{Childrens, ExprRef, PlanArena, TableArenaCell}; - use crate::storage::rocksdb::RocksStorage; + use crate::storage::TestStorage; use crate::storage::{Storage, TableCache}; use crate::types::evaluator::binary_create; use crate::types::value::DataValue; @@ -154,16 +156,17 @@ mod tests { })) } - fn build_test_storage() -> Result<(TableCache, TempDir, RocksStorage), DatabaseError> { + async fn build_test_storage() -> Result<(TableCache, TempDir, TestStorage), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + let storage = crate::storage::new_test_storage().await; Ok((TableCache::default(), temp_dir, storage)) } // Values ignores runtime probes, so it stands in for the unique lookup and the pushed // probes stay on the stack for inspection. #[test] - fn index_probe_join_moves_single_match_and_reuses_right_slots() -> Result<(), DatabaseError> { + async fn index_probe_join_moves_single_match_and_reuses_right_slots( + ) -> Result<(), DatabaseError> { let table_arena = TableArenaCell::default(); let mut plan_arena = PlanArena::new(&table_arena); let mut left = build_values( @@ -197,7 +200,7 @@ mod tests { plan_arena.alloc_expression(ScalarExpression::column_expr(right_schema[1], 3)); let probe = plan_arena.alloc_expression(ScalarExpression::column_expr(left_column, 0)); let op = IndexProbeJoinOperator::new(probe, vec![equality, residual]); - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let cache = &table_cache; let mut exec_buffer = crate::execution::ExecBuffer::default(); @@ -210,7 +213,7 @@ mod tests { ); let mut plan_arena = ExecMetaArena::new(plan_arena); for payload in ["first", "again"] { - assert!(root.next_tuple(&mut arena, &mut plan_arena)?); + assert!(root.next_tuple(&mut arena, &mut plan_arena).await?); assert_eq!( arena.take_tuple(), vec![ @@ -221,7 +224,7 @@ mod tests { ] ); } - assert!(!root.next_tuple(&mut arena, &mut plan_arena)?); + assert!(!root.next_tuple(&mut arena, &mut plan_arena).await?); // The NULL key is skipped without probing. assert_eq!( arena.buffer.runtime_probe_stack, @@ -231,7 +234,7 @@ mod tests { } #[test] - fn index_probe_join_skips_a_rejected_unique_match() -> Result<(), DatabaseError> { + async fn index_probe_join_skips_a_rejected_unique_match() -> Result<(), DatabaseError> { let table_arena = TableArenaCell::default(); let mut plan_arena = PlanArena::new(&table_arena); let mut left = build_values( @@ -249,7 +252,7 @@ mod tests { let predicate = plan_arena.alloc_expression(ScalarExpression::column_expr(right_column, 1)); let probe = plan_arena.alloc_expression(ScalarExpression::column_expr(left_column, 0)); let op = IndexProbeJoinOperator::new(probe, vec![predicate]); - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let mut executor = execute_input::<_, IndexProbeJoin>( (&op, &left, &right), @@ -257,7 +260,7 @@ mod tests { plan_arena, &transaction, ); - assert!(executor.next_tuple()?.is_none()); + assert!(executor.next_tuple().await?.is_none()); Ok(()) } } diff --git a/src/execution/dql/join/nested_loop_join.rs b/src/execution/dql/join/nested_loop_join.rs index 8b6c32c8..02ea3912 100644 --- a/src/execution/dql/join/nested_loop_join.rs +++ b/src/execution/dql/join/nested_loop_join.rs @@ -146,18 +146,19 @@ impl<'a> ReadExecutor<'a> for NestedLoopJoin<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for NestedLoopJoin<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.state = NestedLoopJoinState::PullLeft { right_bitmap: None }; - self.left_input.reset(arena, plan_arena)?; - self.right_input.reset(arena, plan_arena) + self.left_input.reset(arena, plan_arena).await?; + self.right_input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -167,9 +168,9 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for NestedLoopJoin<'a> loop { match state { NestedLoopJoinState::PullLeft { right_bitmap } => { - if !self.left_input.next_tuple(arena, plan_arena)? { + if !self.left_input.next_tuple(arena, plan_arena).await? { if matches!(self.ty, JoinType::Full) { - self.right_input.reset(arena, plan_arena)?; + self.right_input.reset(arena, plan_arena).await?; state = NestedLoopJoinState::EmitRightUnmatched { right_bitmap, right_emit_index: 0, @@ -181,7 +182,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for NestedLoopJoin<'a> return Ok(()); } let left_tuple = arena.take_tuple(); - self.right_input.reset(arena, plan_arena)?; + self.right_input.reset(arena, plan_arena).await?; state = NestedLoopJoinState::ScanRight { active_left: ActiveLeftState { @@ -197,7 +198,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for NestedLoopJoin<'a> mut active_left, mut right_bitmap, } => { - while self.right_input.next_tuple(arena, plan_arena)? { + while self.right_input.next_tuple(arena, plan_arena).await? { let right_tuple = arena.result_tuple(); let idx = active_left.right_index; active_left.right_index += 1; @@ -285,7 +286,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for NestedLoopJoin<'a> right_bitmap, mut right_emit_index, } => { - while self.right_input.next_tuple(arena, plan_arena)? { + while self.right_input.next_tuple(arena, plan_arena).await? { let idx = right_emit_index; right_emit_index += 1; @@ -345,6 +346,7 @@ impl NestedLoopJoin<'_> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; @@ -360,7 +362,6 @@ mod test { use crate::planner::operator::Operator; use crate::planner::test::PlanArenaTestExt; use crate::planner::{Childrens, LogicalPlan}; - use crate::storage::rocksdb::RocksStorage; use crate::storage::Storage; use crate::types::evaluator::binary_create; use crate::types::LogicalType; @@ -533,8 +534,8 @@ mod test { } #[test] - fn nested_right_subtrees_reuse_slots() -> Result<(), DatabaseError> { - let storage = crate::storage::memory::MemoryStorage::new(); + async fn nested_right_subtrees_reuse_slots() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -563,19 +564,18 @@ mod test { let mut root = build_read(&mut arena, &plan_arena, &plan); let mut rows = 0; let mut plan_arena = ExecMetaArena::new(plan_arena); - while root.next_tuple(&mut arena, &mut plan_arena)? { + while root.next_tuple(&mut arena, &mut plan_arena).await? { rows += 1; assert_eq!(arena.result_tuple().len(), 9); } assert_eq!(rows, 64); - assert!(!root.next_tuple(&mut arena, &mut plan_arena)?); + assert!(!root.next_tuple(&mut arena, &mut plan_arena).await?); Ok(()) } #[test] - fn test_nested_inner_join() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_nested_inner_join() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -609,7 +609,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; let mut expected_set = HashSet::with_capacity(1); expected_set.insert(build_integers(vec![ @@ -627,9 +627,8 @@ mod test { } #[test] - fn test_nested_left_out_join() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_nested_left_out_join() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -663,7 +662,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; assert_eq!( tuples[0], @@ -710,9 +709,8 @@ mod test { } #[test] - fn test_nested_cross_join_with_on() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_nested_cross_join_with_on() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -746,7 +744,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; let mut expected_set = HashSet::with_capacity(1); expected_set.insert(build_integers(vec![ @@ -764,9 +762,8 @@ mod test { } #[test] - fn test_nested_cross_join_without_filter() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_nested_cross_join_without_filter() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -800,7 +797,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; let mut expected_set = HashSet::with_capacity(3); expected_set.insert(build_integers(vec![ @@ -833,9 +830,8 @@ mod test { } #[test] - fn test_nested_cross_join_without_on() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_nested_cross_join_without_on() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -869,7 +865,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; assert_eq!(tuples.len(), 16); @@ -877,9 +873,8 @@ mod test { } #[test] - fn test_nested_right_out_join() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_nested_right_out_join() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -913,7 +908,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; let mut expected_set = HashSet::with_capacity(4); expected_set.insert(build_integers(vec![ @@ -955,9 +950,8 @@ mod test { } #[test] - fn test_nested_full_join() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_nested_full_join() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -991,7 +985,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; assert_eq!( tuples[0], @@ -1062,9 +1056,8 @@ mod test { } #[test] - fn test_nested_right_join_filter_only_left_columns() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_nested_right_join_filter_only_left_columns() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = crate::storage::TableCache::default(); let table_arena = crate::planner::TableArenaCell::default(); @@ -1149,7 +1142,7 @@ mod test { plan_arena, &transaction, ); - let tuples = try_collect(executor)?; + let tuples = try_collect(executor).await?; assert_eq!(tuples.len(), 1); assert_eq!( @@ -1165,9 +1158,12 @@ mod test { } #[test] - fn test_right_join_using_binds_visible_column_to_right_side() -> Result<(), DatabaseError> { + async fn test_right_join_using_binds_visible_column_to_right_side() -> Result<(), DatabaseError> + { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut db = DataBaseBuilder::path(temp_dir.path()).build_in_memory()?; + let mut db = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; let setup_sql = [ "DROP TABLE IF EXISTS str1", @@ -1180,28 +1176,30 @@ mod test { for sql in setup_sql { if sql.starts_with("DROP ") || sql.starts_with("CREATE ") { - db.ddl(sql)?; + db.ddl(sql).await?; if sql.starts_with("CREATE TABLE str1") { - db.load(CatalogKind::Table("str1".to_string()))?; + db.load(CatalogKind::Table("str1".to_string())).await?; } else if sql.starts_with("CREATE TABLE str2") { - db.load(CatalogKind::Table("str2".to_string()))?; + db.load(CatalogKind::Table("str2".to_string())).await?; } } else { - db.run(sql)?.done()?; + db.run(sql).await?.done().await?; } } - let mut iter = db.run( - "SELECT s, str1.s, str2.s \ + let mut iter = db + .run( + "SELECT s, str1.s, str2.s \ FROM str1 RIGHT OUTER JOIN str2 USING(s) \ ORDER BY str2.s", - )?; + ) + .await?; let mut actual = Vec::new(); - while let Some(row) = iter.next_tuple(|_, tuple| tuple_to_strings(tuple))? { + while let Some(row) = iter.next_tuple(|_, tuple| tuple_to_strings(tuple)).await? { actual.push(row); } - iter.done()?; + iter.done().await?; assert_eq!( actual, diff --git a/src/execution/dql/limit.rs b/src/execution/dql/limit.rs index 38f7cec1..c735baf5 100644 --- a/src/execution/dql/limit.rs +++ b/src/execution/dql/limit.rs @@ -47,18 +47,19 @@ impl<'a> ReadExecutor<'a> for Limit<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Limit<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.skipped = 0; self.emitted = 0; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -72,7 +73,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Limit<'a> { } loop { - if !self.input.next_tuple(arena, plan_arena)? { + if !self.input.next_tuple(arena, plan_arena).await? { arena.finish(); return Ok(()); } diff --git a/src/execution/dql/mark_apply.rs b/src/execution/dql/mark_apply.rs index d9cb094e..e750768a 100644 --- a/src/execution/dql/mark_apply.rs +++ b/src/execution/dql/mark_apply.rs @@ -58,6 +58,7 @@ enum RightInput { Materialized(MaterializedKeys), } +#[maybe_async] impl RightInput { fn new(materialize: bool) -> Self { if materialize { @@ -68,7 +69,7 @@ impl RightInput { } // Return the first terminal predicate result, or None after exhausting the RHS. - fn scan<'a, T: StorageTransaction + 'a, A: MetaArena + 'a>( + async fn scan<'a, T: StorageTransaction + 'a, A: MetaArena + 'a>( root: &mut ExecNode<'a>, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -81,8 +82,8 @@ impl RightInput { if let Some(probe) = probe { arena.push_runtime_probe(probe); } - root.reset(arena, plan_arena)?; - while root.next_tuple(arena, plan_arena)? { + root.reset(arena, plan_arena).await?; + while root.next_tuple(arena, plan_arena).await? { if let Some(value) = evaluate(arena.result_tuple(), plan_arena)? { return Ok(Some(value)); } @@ -118,8 +119,9 @@ impl<'a> ReadExecutor<'a> for MarkApply<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -129,10 +131,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { if arena.restarting() || !self.op.materialize || self.right_input.is_none() { self.right_input = Some(RightInput::new(self.op.materialize)); } - self.left_input.reset(arena, plan_arena) + self.left_input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -141,7 +143,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { arena.finish(); return Ok(()); }; - if !self.left_input.next_tuple(arena, plan_arena)? { + if !self.left_input.next_tuple(arena, plan_arena).await? { // Return the temporarily owned cache so reset can reuse it after EOF. if self.op.materialize { self.right_input = Some(input); @@ -157,7 +159,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { { (input, DataValue::Null) } - _ => self.mark_value(input, arena, plan_arena, &left)?, + _ => self.mark_value(input, arena, plan_arena, &left).await?, }; self.right_input = Some(input); left.push(marker); @@ -166,6 +168,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for MarkApply<'a> { } } +#[maybe_async] impl<'a> MarkApply<'a> { fn runtime_probe_for(&self, param_value: Option) -> Option { self.op.parameterized_probe()?; @@ -203,7 +206,7 @@ impl<'a> MarkApply<'a> { .transpose() } - fn mark_value( + async fn mark_value( &mut self, mut input: RightInput, arena: &mut ExecContext<'_, 'a, '_, T>, @@ -224,22 +227,26 @@ impl<'a> MarkApply<'a> { Ok(predicates_matched(predicates, left_tuple, right, meta)? .then_some(DataValue::Boolean(true))) }, - )? + ) + .await? .unwrap_or(DataValue::Boolean(false)) } MarkApplyKind::Quantified(quantifier) => { if let RightInput::Materialize = input { input = RightInput::Materialized( - self.materialize_keys(arena, plan_arena, left_tuple)?, + self.materialize_keys(arena, plan_arena, left_tuple).await?, ); } match &input { RightInput::Materialized(keys) => { self.lookup_keys(keys, quantifier, plan_arena, left_tuple)? } - _ => self.scan_quantified_right_input( - arena, plan_arena, probe, quantifier, left_tuple, - )?, + _ => { + self.scan_quantified_right_input( + arena, plan_arena, probe, quantifier, left_tuple, + ) + .await? + } } } }; @@ -267,7 +274,7 @@ impl<'a> MarkApply<'a> { Ok((*op, *left_expr, *right_expr, evaluator)) } - fn materialize_keys( + async fn materialize_keys( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -275,10 +282,10 @@ impl<'a> MarkApply<'a> { ) -> Result { let (_, _, right_expr, _) = Self::quantified_comparison(self.op.predicates(), plan_arena)?; let root = &mut self.right_root; - root.reset(arena, plan_arena)?; + root.reset(arena, plan_arena).await?; let mut builder = KeysBuilder::new(compare_keys); let (mut is_empty, mut has_null) = (true, false); - while root.next_tuple(arena, plan_arena)? { + while root.next_tuple(arena, plan_arena).await? { // RHS positions follow the outer row, so evaluate against the joined layout. let values = SplitTupleRef::new(left_tuple, arena.result_tuple()); let key = plan_arena @@ -358,7 +365,7 @@ impl<'a> MarkApply<'a> { }) } - fn scan_quantified_right_input( + async fn scan_quantified_right_input( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -392,7 +399,8 @@ impl<'a> MarkApply<'a> { } Ok(None) }, - )?; + ) + .await?; if let Some(value) = value { return Ok(value); } @@ -417,6 +425,7 @@ impl<'a> MarkApply<'a> { }; std::iter::once(first).chain(null_keys) } + fn quantified_row_value( predicates: &[ExprRef], left_tuple: &TupleImpl, @@ -442,6 +451,7 @@ impl<'a> MarkApply<'a> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; @@ -451,7 +461,7 @@ mod tests { use crate::planner::operator::Operator; use crate::planner::test::PlanArenaTestExt; use crate::planner::{Childrens, ExprRef, LogicalPlan}; - use crate::storage::rocksdb::RocksStorage; + use crate::storage::TestStorage; use crate::storage::{Storage, TableCache}; use crate::types::evaluator::{binary_create, cast_create}; use crate::types::index::RuntimeIndexProbe; @@ -494,11 +504,11 @@ mod tests { build_values_with_schema(arena, vec![(name, LogicalType::Integer)], rows) } - fn build_test_storage() -> Result<(TableCache, TempDir, RocksStorage), DatabaseError> { + async fn build_test_storage() -> Result<(TableCache, TempDir, TestStorage), DatabaseError> { let table_cache = crate::storage::TableCache::default(); let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + let storage = crate::storage::new_test_storage().await; Ok((table_cache, temp_dir, storage)) } @@ -535,7 +545,7 @@ mod tests { } #[test] - fn mark_exists_apply_appends_boolean_match_column() -> Result<(), DatabaseError> { + async fn mark_exists_apply_appends_boolean_match_column() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let mut left = build_values( @@ -555,7 +565,7 @@ mod tests { let op = MarkApplyOperator::new_exists(build_marker_column(&mut plan_arena), vec![predicate]); - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let context = &table_cache; let mut exec_buffer = crate::execution::ExecBuffer::default(); @@ -568,7 +578,7 @@ mod tests { ); let mut tuples = Vec::new(); let mut plan_arena = ExecMetaArena::new(plan_arena); - while root.next_tuple(&mut arena, &mut plan_arena)? { + while root.next_tuple(&mut arena, &mut plan_arena).await? { tuples.push(arena.take_tuple()); } @@ -590,7 +600,7 @@ mod tests { } #[test] - fn mark_exists_apply_treats_null_predicate_as_not_matched() -> Result<(), DatabaseError> { + async fn mark_exists_apply_treats_null_predicate_as_not_matched() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let mut left = build_values( @@ -610,14 +620,15 @@ mod tests { let op = MarkApplyOperator::new_exists(build_marker_column(&mut plan_arena), vec![predicate]); - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let tuples = try_collect(execute_input::<_, MarkApply>( (&op, &left, &right), &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; assert_eq!( tuples @@ -636,8 +647,8 @@ mod tests { } #[test] - fn mark_exists_apply_sets_runtime_probe_before_residual_predicates() -> Result<(), DatabaseError> - { + async fn mark_exists_apply_sets_runtime_probe_before_residual_predicates( + ) -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let mut left = build_values_with_schema( @@ -678,7 +689,7 @@ mod tests { plan_arena.alloc_expression(ScalarExpression::column_expr(left_value_column, 0)); op.set_parameterized_probe(Some(probe)); - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let mut exec_buffer = crate::execution::ExecBuffer::default(); let mut exec_arena = crate::execution::ExecArena::new(&table_cache, &transaction); @@ -697,7 +708,8 @@ mod tests { let mut plan_arena = ExecMetaArena::new(plan_arena); let input = exec.right_input.take().unwrap(); assert_eq!( - exec.mark_value(input, &mut arena, &mut plan_arena, &left_tuple)? + exec.mark_value(input, &mut arena, &mut plan_arena, &left_tuple) + .await? .1, DataValue::Boolean(true) ); @@ -710,7 +722,7 @@ mod tests { } #[test] - fn mark_in_apply_sets_eq_runtime_probe_for_non_null_value() -> Result<(), DatabaseError> { + async fn mark_in_apply_sets_eq_runtime_probe_for_non_null_value() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let mut left = build_values_with_schema( @@ -733,7 +745,7 @@ mod tests { plan_arena.alloc_expression(ScalarExpression::column_expr(left_value_column, 0)); op.set_parameterized_probe(Some(probe)); - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let mut exec_buffer = crate::execution::ExecBuffer::default(); let mut exec_arena = crate::execution::ExecArena::new(&table_cache, &transaction); @@ -751,7 +763,8 @@ mod tests { let mut plan_arena = ExecMetaArena::new(plan_arena); let input = exec.right_input.take().unwrap(); assert_eq!( - exec.mark_value(input, &mut arena, &mut plan_arena, &left_tuple)? + exec.mark_value(input, &mut arena, &mut plan_arena, &left_tuple) + .await? .1, DataValue::Boolean(true) ); @@ -764,7 +777,7 @@ mod tests { } #[test] - fn mark_in_apply_sets_scope_runtime_probe_for_null_value() -> Result<(), DatabaseError> { + async fn mark_in_apply_sets_scope_runtime_probe_for_null_value() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let mut left = build_values_with_schema( @@ -787,7 +800,7 @@ mod tests { plan_arena.alloc_expression(ScalarExpression::column_expr(left_value_column, 0)), )); - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let mut exec_buffer = crate::execution::ExecBuffer::default(); let mut exec_arena = crate::execution::ExecArena::new(&table_cache, &transaction); @@ -805,7 +818,8 @@ mod tests { let mut plan_arena = ExecMetaArena::new(plan_arena); let input = exec.right_input.take().unwrap(); assert_eq!( - exec.mark_value(input, &mut arena, &mut plan_arena, &left_tuple)? + exec.mark_value(input, &mut arena, &mut plan_arena, &left_tuple) + .await? .1, DataValue::Null ); @@ -821,7 +835,7 @@ mod tests { } #[test] - fn mark_in_apply_appends_boolean_match_column() -> Result<(), DatabaseError> { + async fn mark_in_apply_appends_boolean_match_column() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let mut left = build_values( @@ -839,7 +853,7 @@ mod tests { let predicate = build_equality_predicate(&mut plan_arena, left_column, 0, right_column, 1)?; - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let tuples = try_collect(execute_input::<_, MarkApply>( ( @@ -850,7 +864,8 @@ mod tests { &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; assert_eq!( tuples @@ -869,8 +884,9 @@ mod tests { } #[test] - fn guard_false_skips_failing_rhs_but_true_and_null_evaluate_it() -> Result<(), DatabaseError> { - let (table_cache, _temp_dir, storage) = build_test_storage()?; + async fn guard_false_skips_failing_rhs_but_true_and_null_evaluate_it( + ) -> Result<(), DatabaseError> { + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let cache = &table_cache; for kind in [ @@ -944,7 +960,7 @@ mod tests { let mut meta = ExecMetaArena::new(meta); let result = exec.next_tuple(&mut arena, &mut meta); if use_guard && flag == DataValue::Boolean(false) { - result?; + result.await?; assert_eq!( *arena.result_tuple(), vec![DataValue::Int32(1), flag, DataValue::Null] @@ -953,10 +969,10 @@ mod tests { exec.right_input, Some(RightInput::Streaming | RightInput::Materialize) )); - exec.next_tuple(&mut arena, &mut meta)?; + exec.next_tuple(&mut arena, &mut meta).await?; assert_eq!(exec.right_input.is_none(), !op.materialize); } else { - assert!(matches!(result, Err(DatabaseError::ParseInt(_)))); + assert!(matches!(result.await, Err(DatabaseError::ParseInt(_)))); } } } @@ -965,8 +981,8 @@ mod tests { } #[test] - fn guard_skips_preserve_materialized_rhs() -> Result<(), DatabaseError> { - let (table_cache, _temp_dir, storage) = build_test_storage()?; + async fn guard_skips_preserve_materialized_rhs() -> Result<(), DatabaseError> { + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let cache = &table_cache; let cases = [ @@ -1027,7 +1043,7 @@ mod tests { }; let mut meta = ExecMetaArena::new(meta); for (index, (key, flag, marker)) in cases.iter().enumerate() { - exec.next_tuple(&mut arena, &mut meta)?; + exec.next_tuple(&mut arena, &mut meta).await?; assert_eq!( *arena.result_tuple(), vec![DataValue::Int32(*key), flag.clone(), marker.clone()] @@ -1046,16 +1062,16 @@ mod tests { assert!(matches!(exec.right_input, Some(RightInput::Streaming))); } } - exec.next_tuple(&mut arena, &mut meta)?; + exec.next_tuple(&mut arena, &mut meta).await?; assert_eq!(exec.right_input.is_none(), !materialize); } Ok(()) } #[test] - fn quantified_apply_materializes_once_and_reuses_keys_after_short_circuit( + async fn quantified_apply_materializes_once_and_reuses_keys_after_short_circuit( ) -> Result<(), DatabaseError> { - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let cache = &table_cache; // Exercise both a single segment and the default 1024-row spill boundary. @@ -1106,7 +1122,9 @@ mod tests { }; let tuple: TupleImpl = Tuple::new(None, vec![value]).into(); let input = exec.right_input.take().unwrap(); - let (input, value) = exec.mark_value(input, &mut arena, &mut meta, &tuple)?; + let (input, value) = exec + .mark_value(input, &mut arena, &mut meta, &tuple) + .await?; exec.right_input = Some(input); assert_eq!(value, expected); // A short-circuited match still caches every RHS key. @@ -1114,21 +1132,21 @@ mod tests { panic!("the first lookup must materialize the RHS") }; assert!(!keys.is_empty && keys.has_null); - exec.reset(&mut arena, &mut meta)?; + exec.reset(&mut arena, &mut meta).await?; assert!(matches!( exec.right_input, Some(RightInput::Materialized(_)) )); } // Exhausting the left must not discard the RHS cache either. - while exec.left_input.next_tuple(&mut arena, &mut meta)? {} - exec.next_tuple(&mut arena, &mut meta)?; + while exec.left_input.next_tuple(&mut arena, &mut meta).await? {} + exec.next_tuple(&mut arena, &mut meta).await?; assert!(matches!( exec.right_input, Some(RightInput::Materialized(_)) )); - exec.reset(&mut arena, &mut meta)?; - exec.next_tuple(&mut arena, &mut meta)?; + exec.reset(&mut arena, &mut meta).await?; + exec.next_tuple(&mut arena, &mut meta).await?; assert_eq!( *arena.result_tuple(), vec![ @@ -1149,12 +1167,14 @@ mod tests { exec.right_root = build_read(&mut arena, &meta, &empty); exec.right_input = Some(RightInput::Materialize); let input = exec.right_input.take().unwrap(); - let (input, value) = exec.mark_value( - input, - &mut arena, - &mut meta, - &TupleImpl::from(Tuple::new(None, vec![DataValue::Null])), - )?; + let (input, value) = exec + .mark_value( + input, + &mut arena, + &mut meta, + &TupleImpl::from(Tuple::new(None, vec![DataValue::Null])), + ) + .await?; exec.right_input = Some(input); assert_eq!( value, @@ -1164,12 +1184,12 @@ mod tests { exec.right_input, Some(RightInput::Materialized(_)) )); - exec.reset(&mut arena, &mut meta)?; + exec.reset(&mut arena, &mut meta).await?; assert!(matches!( exec.right_input, Some(RightInput::Materialized(_)) )); - exec.next_tuple(&mut arena, &mut meta)?; + exec.next_tuple(&mut arena, &mut meta).await?; assert_eq!( arena.result_tuple().iter().last(), Some(&DataValue::Boolean(matches!( @@ -1182,7 +1202,7 @@ mod tests { } #[test] - fn paused_rhs_survives_left_subquery_rebuilds() -> Result<(), DatabaseError> { + async fn paused_rhs_survives_left_subquery_rebuilds() -> Result<(), DatabaseError> { let tables = crate::planner::TableArenaCell::default(); let mut meta = crate::planner::PlanArena::new(&tables); let mut left = build_values( @@ -1220,14 +1240,15 @@ mod tests { let mut op = MarkApplyOperator::new_quantified(MarkApplyQuantifier::Any, marker, vec![predicate]); op.materialize = true; - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let tuples = try_collect(execute_input::<_, MarkApply>( (&op, &left, &right), &table_cache, meta, &transaction, - ))?; + )) + .await?; assert_eq!( tuples .into_iter() @@ -1260,7 +1281,7 @@ mod tests { } #[test] - fn mark_in_apply_treats_null_predicate_as_not_matched() -> Result<(), DatabaseError> { + async fn mark_in_apply_treats_null_predicate_as_not_matched() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let mut left = build_values( @@ -1278,7 +1299,7 @@ mod tests { let predicate = build_equality_predicate(&mut plan_arena, left_column, 0, right_column, 1)?; - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let tuples = try_collect(execute_input::<_, MarkApply>( ( @@ -1289,7 +1310,8 @@ mod tests { &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; assert_eq!( tuples @@ -1308,7 +1330,7 @@ mod tests { } #[test] - fn mark_in_apply_ignores_null_correlated_predicate_rows() -> Result<(), DatabaseError> { + async fn mark_in_apply_ignores_null_correlated_predicate_rows() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = crate::planner::PlanArena::new(&table_arena); let mut left = build_values( @@ -1346,7 +1368,7 @@ mod tests { ty: LogicalType::Boolean, }); - let (table_cache, _temp_dir, storage) = build_test_storage()?; + let (table_cache, _temp_dir, storage) = build_test_storage().await?; let transaction = storage.transaction()?; let tuples = try_collect(execute_input::<_, MarkApply>( ( @@ -1360,7 +1382,8 @@ mod tests { &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; assert_eq!( tuples diff --git a/src/execution/dql/projection.rs b/src/execution/dql/projection.rs index a73b72ac..ddae74fd 100644 --- a/src/execution/dql/projection.rs +++ b/src/execution/dql/projection.rs @@ -41,21 +41,22 @@ impl<'a> ReadExecutor<'a> for Projection<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Projection<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { - if !self.input.next_tuple(arena, plan_arena)? { + if !self.input.next_tuple(arena, plan_arena).await? { arena.finish(); return Ok(()); } diff --git a/src/execution/dql/recursive_cte.rs b/src/execution/dql/recursive_cte.rs index c4dee99b..728111b5 100644 --- a/src/execution/dql/recursive_cte.rs +++ b/src/execution/dql/recursive_cte.rs @@ -220,6 +220,7 @@ pub struct RecursiveCte<'a> { phase: RecursivePhase, } +#[maybe_async] impl<'a> RecursiveCte<'a> { fn new(anchor_input: ExecNode<'a>, recursive_plan: &'a LogicalPlan) -> Self { Self { @@ -233,7 +234,7 @@ impl<'a> RecursiveCte<'a> { } } - fn start_recursive( + async fn start_recursive( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -262,8 +263,9 @@ impl<'a> ReadExecutor<'a> for RecursiveCte<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for RecursiveCte<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -274,10 +276,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for RecursiveCte<'a> { if self.recursive_root.take().is_some() { arena.truncate_iters(self.iter_mark); } - self.anchor_input.reset(arena, plan_arena) + self.anchor_input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -285,7 +287,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for RecursiveCte<'a> { loop { match self.phase { RecursivePhase::Anchor => { - while self.anchor_input.next_tuple(arena, plan_arena)? { + while self.anchor_input.next_tuple(arena, plan_arena).await? { self.next.push(arena.take_tuple())?; } self.working = mem::take(&mut self.next).finish()?; @@ -296,7 +298,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for RecursiveCte<'a> { arena.produce(tuple); return Ok(()); } - if !self.start_recursive(arena, plan_arena)? { + if !self.start_recursive(arena, plan_arena).await? { arena.finish(); return Ok(()); } @@ -307,7 +309,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for RecursiveCte<'a> { .recursive_root .as_mut() .expect("recursive step is built before it runs"); - while root.next_tuple(arena, plan_arena)? { + while root.next_tuple(arena, plan_arena).await? { self.next.push(arena.take_tuple())?; } self.recursive_root = None; @@ -337,8 +339,9 @@ impl<'a> ReadExecutor<'a> for RecursiveScan { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for RecursiveScan { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -346,7 +349,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for RecursiveScan { self.input.reset() } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -361,6 +364,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for RecursiveScan { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; @@ -374,13 +378,11 @@ mod tests { use crate::planner::test::PlanArenaTestExt; use crate::planner::Childrens; use crate::planner::LogicalPlan; - use crate::storage::rocksdb::RocksStorage; use crate::storage::{Storage, TableCache}; use crate::types::evaluator::binary_create; use crate::types::value::DataValue; use crate::types::LogicalType; use std::borrow::Cow; - use tempfile::TempDir; #[cfg(feature = "spill")] #[test] @@ -449,7 +451,7 @@ mod tests { } #[test] - fn recursive_cte_replays_each_generation() -> Result<(), DatabaseError> { + async fn recursive_cte_replays_each_generation() -> Result<(), DatabaseError> { let table_arena = crate::planner::TableArenaCell::default(); let mut plan_arena = PlanArena::new(&table_arena); let column = plan_arena.alloc_column(ColumnCatalog::new( @@ -500,8 +502,7 @@ mod tests { Childrens::Only(Box::new(filter)), ); - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let table_cache = TableCache::default(); let tuples = try_collect(execute_input::<_, RecursiveCte<'_>>( @@ -509,7 +510,8 @@ mod tests { &table_cache, plan_arena, &transaction, - ))?; + )) + .await?; assert_eq!( tuples diff --git a/src/execution/dql/scalar_query_init.rs b/src/execution/dql/scalar_query_init.rs index 2805b625..d63c2e3e 100644 --- a/src/execution/dql/scalar_query_init.rs +++ b/src/execution/dql/scalar_query_init.rs @@ -42,17 +42,18 @@ pub struct ScalarQueryInit<'a> { init: ExecNode<'a>, } +#[maybe_async] impl<'a> ScalarQueryInit<'a> { - fn evaluate_init( + async fn evaluate_init( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { let mut value = DataValue::Null; - if self.init.next_tuple(arena, plan_arena)? { + if self.init.next_tuple(arena, plan_arena).await? { std::mem::swap(&mut value, &mut arena.result_mut()[0]); } - if self.init.next_tuple(arena, plan_arena)? { + if self.init.next_tuple(arena, plan_arena).await? { return Err(DatabaseError::InvalidValue( "scalar subquery returned more than one row".into(), )); @@ -104,8 +105,9 @@ impl<'a> ReadExecutor<'a> for ScalarQueryInit<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ScalarQueryInit<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -116,10 +118,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ScalarQueryInit<'a> ScalarQueryInitState::Initialize }; self.scratch_tuple.output_mut().clear(); - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -128,19 +130,19 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ScalarQueryInit<'a> match self.state { ScalarQueryInitState::Initialize => { if plan_arena.init_value(self.reference).is_none() { - self.init.reset(arena, plan_arena)?; - self.evaluate_init(arena, plan_arena)?; + self.init.reset(arena, plan_arena).await?; + self.evaluate_init(arena, plan_arena).await?; } self.state = ScalarQueryInitState::ReadInput; } ScalarQueryInitState::ReadInput => { - if !self.input.next_tuple(arena, plan_arena)? { + if !self.input.next_tuple(arena, plan_arena).await? { self.state = ScalarQueryInitState::Finished; } return Ok(()); } ScalarQueryInitState::ReadOuter => { - if !self.input.next_tuple(arena, plan_arena)? { + if !self.input.next_tuple(arena, plan_arena).await? { self.state = ScalarQueryInitState::Finished; return Ok(()); } @@ -156,8 +158,8 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ScalarQueryInit<'a> self.state = ScalarQueryInitState::EvaluateOuter; } ScalarQueryInitState::EvaluateOuter => { - self.init.reset(arena, plan_arena)?; - self.evaluate_init(arena, plan_arena)?; + self.init.reset(arena, plan_arena).await?; + self.evaluate_init(arena, plan_arena).await?; std::mem::swap(arena.result_mut(), &mut self.scratch_tuple); self.state = ScalarQueryInitState::ReadOuter; arena.resume(); @@ -174,6 +176,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ScalarQueryInit<'a> // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; @@ -182,13 +185,13 @@ mod tests { use crate::planner::operator::values::ValuesOperator; use crate::planner::operator::Operator; use crate::planner::{Childrens, ExecMetaArena, PlanArena, TableArenaCell}; - use crate::storage::memory::MemoryStorage; use crate::storage::{Storage, TableCache}; use crate::types::LogicalType; #[test] - fn scalar_init_shared_slot_caches_null_and_checks_cardinality() -> Result<(), DatabaseError> { - let storage = MemoryStorage::new(); + async fn scalar_init_shared_slot_caches_null_and_checks_cardinality( + ) -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let tables = TableCache::default(); let cache = &tables; @@ -238,13 +241,13 @@ mod tests { let mut duplicate_root = build_read(&mut arena, &metadata, &duplicate); let mut view = ExecMetaArena::new(metadata); if row_count == 2 { - assert!(first_root.next_tuple(&mut arena, &mut view).is_err()); + assert!(first_root.next_tuple(&mut arena, &mut view).await.is_err()); assert_eq!(view.init_value(reference), None); } else { - assert!(first_root.next_tuple(&mut arena, &mut view)?); + assert!(first_root.next_tuple(&mut arena, &mut view).await?); assert_eq!(view.init_value(reference), Some(&value)); - assert!(duplicate_root.next_tuple(&mut arena, &mut view)?); - assert!(!duplicate_root.next_tuple(&mut arena, &mut view)?); + assert!(duplicate_root.next_tuple(&mut arena, &mut view).await?); + assert!(!duplicate_root.next_tuple(&mut arena, &mut view).await?); } } Ok(()) diff --git a/src/execution/dql/scalar_subquery.rs b/src/execution/dql/scalar_subquery.rs index 491dcb66..5b2903c0 100644 --- a/src/execution/dql/scalar_subquery.rs +++ b/src/execution/dql/scalar_subquery.rs @@ -46,22 +46,23 @@ impl<'a> ReadExecutor<'a> for ScalarSubquery<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ScalarSubquery<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.returned = false; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { - let has_next = self.input.next_tuple(arena, plan_arena)?; + let has_next = self.input.next_tuple(arena, plan_arena).await?; if self.returned { if has_next { return Err(DatabaseError::InvalidValue( diff --git a/src/execution/dql/seq_scan.rs b/src/execution/dql/seq_scan.rs index 6e06c206..bc69d245 100644 --- a/src/execution/dql/seq_scan.rs +++ b/src/execution/dql/seq_scan.rs @@ -18,7 +18,6 @@ use crate::execution::{ }; use crate::planner::operator::table_scan::TableScanOperator; use crate::planner::{ExecMetaArena, MetaArena}; -use crate::storage::table_codec::TableCodec; use crate::storage::{Iter, StorageTransaction}; use crate::types::tuple::Tuple; @@ -40,8 +39,9 @@ impl<'a> ReadExecutor<'a> for SeqScan<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SeqScan<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -51,12 +51,13 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SeqScan<'a> { let iter = &mut state.iters[self.iter]; if !iter.is_empty() { iter.as_tuple() - .reset(transaction, state.table_codec, self.op.limit)?; + .reset(transaction, state.table_codec, self.op.limit) + .await?; } Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -69,21 +70,23 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SeqScan<'a> { limit, .. } = self.op; - let iter = - state - .transaction() - .read(state.table_codec, deserializers, *table_id, *limit)?; + let iter = state + .transaction() + .read(state.table_codec, deserializers, *table_id, *limit) + .await?; state.iters[self.iter] = StorageIter::Tuple(iter); } let iter = state.iters[self.iter].as_tuple(); let output = state.result.output_mut(); - if next_scan_tuple( - self.op, - |table_codec, tuple| iter.next_tuple_into(table_codec, tuple), - state.table_codec, - output, - plan_arena, - )? { + let found = loop { + if !iter.next_tuple_into(state.table_codec, output).await? { + break false; + } + if scan_predicate_holds(self.op, output, plan_arena)? { + break true; + } + }; + if found { arena.resume(); } else { arena.finish(); @@ -93,35 +96,27 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SeqScan<'a> { } #[inline] -pub(crate) fn next_scan_tuple( +pub(crate) fn scan_predicate_holds( op: &TableScanOperator, - mut next: impl FnMut(&mut TableCodec, &mut Tuple) -> Result, - table_codec: &mut TableCodec, - tuple: &mut Tuple, + tuple: &Tuple, arena: &A, ) -> Result { - while next(table_codec, tuple)? { - let holds = match op.predicate { - Some(predicate) => arena - .expression(predicate) - .eval(arena, Some(&*tuple))? - .is_true()?, - None => true, - }; - if holds { - return Ok(true); - } + match op.predicate { + Some(predicate) => arena + .expression(predicate) + .eval(arena, Some(tuple))? + .is_true(), + None => Ok(true), } - Ok(false) } // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::db::test::build_table; use crate::planner::{PlanArena, TableArenaCell}; - use crate::storage::memory::MemoryStorage; use crate::storage::table_codec::TableCodec; use crate::storage::{Storage, TableCache, TupleDeserializers}; use crate::types::tuple::Tuple; @@ -129,14 +124,14 @@ mod tests { use crate::types::LogicalType; #[test] - fn reset_keeps_seq_iterator_and_restores_offset_limit() -> Result<(), DatabaseError> { - let storage = MemoryStorage::new(); + async fn reset_keeps_seq_iterator_and_restores_offset_limit() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let mut tx = storage.transaction()?; let mut names = crate::storage::NameMapping::default(); let mut tables = TableCache::default(); let cell = TableArenaCell::default(); let mut meta = PlanArena::new(&cell); - build_table(&mut names, &mut tables, &mut tx, &mut meta)?; + build_table(&mut names, &mut tables, &mut tx, &mut meta).await?; let meta = PlanArena::new(&cell); let serializers = [ LogicalType::Integer.serializable(), @@ -159,7 +154,8 @@ mod tests { .into(), &serializers, false, - )?; + ) + .await?; } let op = TableScanOperator { table_id: 0, @@ -181,18 +177,18 @@ mod tests { op: &op, iter: arena.alloc_iter(), }; - scan.next_tuple(&mut arena, &mut meta)?; + scan.next_tuple(&mut arena, &mut meta).await?; assert_eq!(arena.result_tuple()[0], DataValue::Int32(2)); let iterator = &arena.arena.iters[scan.iter] as *const _; for _ in 0..3 { - scan.reset(&mut arena, &mut meta)?; - scan.reset(&mut arena, &mut meta)?; + scan.reset(&mut arena, &mut meta).await?; + scan.reset(&mut arena, &mut meta).await?; for id in [2, 3] { - scan.next_tuple(&mut arena, &mut meta)?; + scan.next_tuple(&mut arena, &mut meta).await?; assert_eq!(arena.result_tuple()[0], DataValue::Int32(id)); assert_eq!(&arena.arena.iters[scan.iter] as *const _, iterator); } - scan.next_tuple(&mut arena, &mut meta)?; + scan.next_tuple(&mut arena, &mut meta).await?; assert_eq!( arena.buffer.result.status, Some(crate::execution::ExecStatus::End) diff --git a/src/execution/dql/set_membership.rs b/src/execution/dql/set_membership.rs index a6fd7cfb..5bd5815e 100644 --- a/src/execution/dql/set_membership.rs +++ b/src/execution/dql/set_membership.rs @@ -63,8 +63,9 @@ impl<'a> ReadExecutor<'a> for SetMembership<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SetMembership<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -72,17 +73,17 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SetMembership<'a> { self.built = false; self.right_rows.clear(); self.right_counts.clear(); - self.left_input.reset(arena, plan_arena)?; - self.right_input.reset(arena, plan_arena) + self.left_input.reset(arena, plan_arena).await?; + self.right_input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if !self.built { - while self.right_input.next_tuple(arena, plan_arena)? { + while self.right_input.next_tuple(arena, plan_arena).await? { let tuple = arena.result_tuple(); let width = tuple.len(); let hash = hash_row(&self.hasher, tuple.iter()); @@ -104,7 +105,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for SetMembership<'a> { } loop { - if !self.left_input.next_tuple(arena, plan_arena)? { + if !self.left_input.next_tuple(arena, plan_arena).await? { arena.finish(); return Ok(()); } diff --git a/src/execution/dql/show_table.rs b/src/execution/dql/show_table.rs index 5744d935..d4a9910c 100644 --- a/src/execution/dql/show_table.rs +++ b/src/execution/dql/show_table.rs @@ -38,8 +38,9 @@ impl<'a> ReadExecutor<'a> for ShowTables { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ShowTables { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -48,7 +49,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ShowTables { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -56,12 +57,12 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ShowTables { if arena.iter_mut(self.metas).is_empty() { let mut state = arena.local_state(plan_arena); let (transaction, table_codec) = state.transaction_codec(); - let metas = transaction.tables(table_codec)?; + let metas = transaction.tables(table_codec).await?; state.iters[self.metas] = StorageIter::Table(metas); } let metas = arena.iter_mut(self.metas).as_table(); - let Some(table) = metas.try_next(plan_arena)? else { + let Some(table) = metas.try_next(plan_arena).await? else { arena.finish(); return Ok(()); }; diff --git a/src/execution/dql/show_view.rs b/src/execution/dql/show_view.rs index 0a74642b..bf16b035 100644 --- a/src/execution/dql/show_view.rs +++ b/src/execution/dql/show_view.rs @@ -38,8 +38,9 @@ impl<'a> ReadExecutor<'a> for ShowViews { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ShowViews { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -48,7 +49,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ShowViews { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -56,12 +57,12 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for ShowViews { if arena.iter_mut(self.metas).is_empty() { let mut state = arena.local_state(plan_arena); let (transaction, table_codec) = state.transaction_codec(); - let metas = transaction.views(table_codec)?; + let metas = transaction.views(table_codec).await?; state.iters[self.metas] = StorageIter::View(metas); } let metas = arena.iter_mut(self.metas).as_view(); - let Some(name) = metas.try_next(plan_arena)? else { + let Some(name) = metas.try_next(plan_arena).await? else { arena.finish(); return Ok(()); }; diff --git a/src/execution/dql/sort.rs b/src/execution/dql/sort.rs index 25a2008e..15d56954 100644 --- a/src/execution/dql/sort.rs +++ b/src/execution/dql/sort.rs @@ -14,7 +14,7 @@ use crate::errors::DatabaseError; use crate::execution::{ - build_read, ExecBuilder, ExecContext, ExecNode, ExecutorNode, ReadExecutor, + build_read, ArenaBound, ExecBuilder, ExecContext, ExecNode, ExecutorNode, ReadExecutor, }; use crate::planner::operator::sort::{SortField, SortOperator}; use crate::planner::LogicalPlan; @@ -156,7 +156,7 @@ pub(crate) struct SortTuple { } pub struct Sort<'a> { - rows: NullableVec<'static, (usize, SortTuple)>, + rows: ArenaBound>, _arena: Box, sort_fields: &'a [SortField], input: ExecNode<'a>, @@ -178,7 +178,7 @@ impl<'a> ReadExecutor<'a> for Sort<'a> { ) }; ExecNode::Sort(Box::new(Sort { - rows, + rows: ArenaBound(rows), _arena: sort_arena, sort_fields, input, @@ -186,17 +186,18 @@ impl<'a> ReadExecutor<'a> for Sort<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Sort<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.rows.clear(); - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -207,7 +208,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Sort<'a> { arena.produce(row.tuple); return Ok(()); } - while self.input.next_tuple(arena, plan_arena)? { + while self.input.next_tuple(arena, plan_arena).await? { let offset = self.rows.len(); self.rows.put(( offset, @@ -229,6 +230,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Sort<'a> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::errors::DatabaseError; @@ -243,7 +245,7 @@ mod test { use std::cell::Cell; #[test] - fn memory_sort_restores_scalar_values_for_each_output_row() -> Result<(), DatabaseError> { + async fn memory_sort_restores_scalar_values_for_each_output_row() -> Result<(), DatabaseError> { use super::Sort; use crate::execution::ReadExecutor; use crate::planner::operator::scalar_query_init::ScalarQueryInitOperator; @@ -253,10 +255,9 @@ mod test { use crate::planner::{ Childrens, ExecMetaArena, LogicalPlan, MetaArena, PlanArena, TableArenaCell, }; - use crate::storage::memory::MemoryStorage; use crate::storage::{Storage, TableCache}; - let storage = MemoryStorage::new(); + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let tables = TableCache::default(); let cache = &tables; @@ -308,11 +309,11 @@ mod test { // Select the memory executor explicitly even when the spill feature is enabled. let mut root = ::into_executor((&op, &plan), &mut arena, &metadata); let mut metadata = ExecMetaArena::new(metadata); - assert!(root.next_tuple(&mut arena, &mut metadata)?); + assert!(root.next_tuple(&mut arena, &mut metadata).await?); assert_eq!(*arena.result_tuple(), vec![DataValue::Int32(1)]); - root.reset(&mut arena, &mut metadata)?; + root.reset(&mut arena, &mut metadata).await?; for expected in [1, 2, 3] { - assert!(root.next_tuple(&mut arena, &mut metadata)?); + assert!(root.next_tuple(&mut arena, &mut metadata).await?); assert_eq!(*arena.result_tuple(), vec![DataValue::Int32(expected)]); assert_eq!( metadata.init_value(result), @@ -326,17 +327,17 @@ mod test { &DataValue::Int32(expected) ); } - assert!(!root.next_tuple(&mut arena, &mut metadata)?); - root.reset(&mut arena, &mut metadata)?; + assert!(!root.next_tuple(&mut arena, &mut metadata).await?); + root.reset(&mut arena, &mut metadata).await?; for expected in [1, 2, 3] { - assert!(root.next_tuple(&mut arena, &mut metadata)?); + assert!(root.next_tuple(&mut arena, &mut metadata).await?); assert_eq!(*arena.result_tuple(), vec![DataValue::Int32(expected)]); assert_eq!( metadata.init_value(result), Some(&DataValue::Int32(expected)) ); } - assert!(!root.next_tuple(&mut arena, &mut metadata)?); + assert!(!root.next_tuple(&mut arena, &mut metadata).await?); Ok(()) } diff --git a/src/execution/dql/top_k.rs b/src/execution/dql/top_k.rs index 39cc350c..e610de85 100644 --- a/src/execution/dql/top_k.rs +++ b/src/execution/dql/top_k.rs @@ -15,7 +15,7 @@ use crate::errors::DatabaseError; use crate::execution::dql::sort::BumpVec; use crate::execution::{ - build_read, ExecBuilder, ExecContext, ExecNode, ExecutorNode, ReadExecutor, + build_read, ArenaBound, ExecBuilder, ExecContext, ExecNode, ExecutorNode, ReadExecutor, }; use crate::planner::operator::sort::SortField; use crate::planner::operator::top_k::TopKOperator; @@ -109,7 +109,7 @@ fn top_sort<'a>( } pub struct TopK<'a> { - output: Option>>>, + output: Option>>>>, arena: Box, sort_fields: &'a [SortField], limit: usize, @@ -144,18 +144,19 @@ impl<'a> ReadExecutor<'a> for TopK<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for TopK<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.output = None; self.arena.reset(); - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -163,11 +164,11 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for TopK<'a> { if self.output.is_none() { let keep_count = self.offset.unwrap_or(0) + self.limit; #[allow(clippy::mutable_key_type)] - let mut set = BTreeSet::new(); + let mut set = ArenaBound(BTreeSet::new()); let mut sequence = 0; - let mut key_scratch = BumpBytes::new_in(&self.arena); - while self.input.next_tuple(arena, plan_arena)? { + let mut key_scratch = ArenaBound(BumpBytes::new_in(&self.arena)); + while self.input.next_tuple(arena, plan_arena).await? { top_sort( &mut key_scratch, self.sort_fields, @@ -181,18 +182,18 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for TopK<'a> { } let offset = self.offset.unwrap_or(0); - let rows = set.into_iter().skip(offset); + let rows = set.0.into_iter().skip(offset); // The arena lives at a stable boxed address, so we can keep the old set/key shape // and resume iteration across executor polls. - self.output = Some(unsafe { + self.output = Some(ArenaBound(unsafe { transmute::< std::iter::Skip>>, std::iter::Skip>>, >(rows) - }); + })); } - if let Some(item) = self.output.as_mut().and_then(std::iter::Iterator::next) { + if let Some(item) = self.output.as_mut().and_then(|output| output.next()) { plan_arena.restore_outer_values(item.outer_values); arena.produce(item.tuple); } else { @@ -205,6 +206,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for TopK<'a> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] #[allow(clippy::mutable_key_type)] +#[maybe_async] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::errors::DatabaseError; diff --git a/src/execution/dql/union.rs b/src/execution/dql/union.rs index 9f6ebfb3..5838eee7 100644 --- a/src/execution/dql/union.rs +++ b/src/execution/dql/union.rs @@ -43,30 +43,31 @@ impl<'a> ReadExecutor<'a> for Union<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Union<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { self.reading_left = true; - self.left_input.reset(arena, plan_arena)?; - self.right_input.reset(arena, plan_arena) + self.left_input.reset(arena, plan_arena).await?; + self.right_input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { if self.reading_left { - if self.left_input.next_tuple(arena, plan_arena)? { + if self.left_input.next_tuple(arena, plan_arena).await? { arena.resume(); return Ok(()); } self.reading_left = false; } - if self.right_input.next_tuple(arena, plan_arena)? { + if self.right_input.next_tuple(arena, plan_arena).await? { arena.resume(); } else { arena.finish(); diff --git a/src/execution/dql/values.rs b/src/execution/dql/values.rs index 18772dee..625aaabb 100644 --- a/src/execution/dql/values.rs +++ b/src/execution/dql/values.rs @@ -49,8 +49,9 @@ impl<'a> ReadExecutor<'a> for Values<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Values<'a> { - fn reset( + async fn reset( &mut self, _arena: &mut ExecContext<'_, 'a, '_, T>, _plan_arena: &mut ExecMetaArena, @@ -60,7 +61,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Values<'a> { Ok(()) } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, diff --git a/src/execution/dql/window.rs b/src/execution/dql/window.rs index bab68e6f..168681f7 100644 --- a/src/execution/dql/window.rs +++ b/src/execution/dql/window.rs @@ -227,8 +227,9 @@ impl<'a> Window<'a> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Window<'a> { - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -236,10 +237,10 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Window<'a> { self.state.clear(); self.input_exhausted = false; self.reset_functions()?; - self.input.reset(arena, plan_arena) + self.input.reset(arena, plan_arena).await } - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -259,7 +260,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Window<'a> { let (tuple, boundary) = if let Some((tuple, boundary)) = self.state.pending.take() { (tuple, Some(boundary)) - } else if self.input.next_tuple(arena, plan_arena)? { + } else if self.input.next_tuple(arena, plan_arena).await? { (arena.take_tuple(), None) } else { self.eval_functions(plan_arena)?; @@ -275,6 +276,7 @@ impl<'a, T: StorageTransaction + 'a> ExecutorNode<'a, T> for Window<'a> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::ColumnRef; diff --git a/src/execution/dql/window/function.rs b/src/execution/dql/window/function.rs index cb2c3c79..e53943d7 100644 --- a/src/execution/dql/window/function.rs +++ b/src/execution/dql/window/function.rs @@ -16,6 +16,7 @@ use crate::errors::DatabaseError; use crate::execution::dql::aggregate::{create_accumulator, Accumulator}; use crate::expression::agg::AggKind; use crate::expression::window::WindowFunctionKind; +use crate::maybe_async::MaybeSend; use crate::planner::ExprRef; use crate::planner::MetaArena; use crate::types::tuple::TupleImpl; @@ -23,7 +24,7 @@ use crate::types::value::DataValue; use crate::types::LogicalType; use std::ops::Range; -pub(super) trait WindowFunction { +pub(super) trait WindowFunction: MaybeSend { fn reset(&mut self) -> Result<(), DatabaseError> { Ok(()) } diff --git a/src/execution/hash.rs b/src/execution/hash.rs index ad0a1864..261bd6fd 100644 --- a/src/execution/hash.rs +++ b/src/execution/hash.rs @@ -84,6 +84,7 @@ impl Hasher for FxHasher { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::FxBuildHasher; use std::hash::BuildHasher; diff --git a/src/execution/mod.rs b/src/execution/mod.rs index 5d31878e..eeb917a1 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -165,6 +165,7 @@ pub struct Executor<'a> { keeper: PlanKeeper<'a>, } +#[maybe_async] impl<'a> Executor<'a> { pub(crate) fn new(plan: LogicalPlan, plan_arena: &(dyn MetaArena + 'a)) -> Self { let schema = plan.read_schema().clone(); @@ -195,16 +196,16 @@ impl<'a> Executor<'a> { self.iters } - pub(crate) fn next_tuple( + pub(crate) async fn next_tuple( &mut self, arena: &mut ExecArena<'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result { let mut context = ExecContext::new(&mut self.buffer, arena); - self.root.next_tuple(&mut context, plan_arena) + self.root.next_tuple(&mut context, plan_arena).await } - pub(crate) fn start( + pub(crate) async fn start( &mut self, arena: &mut ExecArena<'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -213,7 +214,7 @@ impl<'a> Executor<'a> { if mem::replace(&mut self.ran, true) { self.buffer.restarting = true; let mut context = ExecContext::new(&mut self.buffer, arena); - let result = self.root.reset(&mut context, plan_arena); + let result = self.root.reset(&mut context, plan_arena).await; self.buffer.restarting = false; result?; } @@ -273,174 +274,224 @@ pub(crate) enum ExecNode<'a> { Window(Box>), } +// Nodes are only driven through `ExecNode::next_tuple` / `reset`, which on the async path box +// each future as `dyn Future + Send`; that is where `Send` is checked, against the concrete node. +#[maybe_async] +#[keep_async_fn] pub(crate) trait ExecutorNode<'a, T: StorageTransaction + 'a>: Sized { // Restores the state the node had right after it was built, resetting its inputs too. // A parent may also reset a child to rescan it within one execution; only then may a node // keep what it read (see `ExecContext::restarting`), as a new execution may run in another // transaction. - fn reset( + async fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError>; - fn next_tuple( + async fn next_tuple( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError>; } +#[maybe_async] impl<'a> ExecNode<'a> { + #[sync_only] + #[inline] pub(crate) fn reset( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, + ) -> Result<(), DatabaseError> { + self.reset_inner(arena, plan_arena) + } + + #[sync_only] + #[inline] + pub(crate) fn next_tuple( + &mut self, + arena: &mut ExecContext<'_, 'a, '_, T>, + plan_arena: &mut ExecMetaArena, + ) -> Result { + self.next_tuple_inner(arena, plan_arena) + } + + // The async versions box the node's future: the tree is recursive, so its futures need a + // concrete, sized type. That is also where `Send` is checked, against the concrete node. + #[async_only] + pub(crate) fn reset<'s, T: StorageTransaction + 'a, A: MetaArena + 'a>( + &'s mut self, + arena: &'s mut ExecContext<'_, 'a, '_, T>, + plan_arena: &'s mut ExecMetaArena, + ) -> std::pin::Pin> + Send + 's>> + { + Box::pin(self.reset_inner(arena, plan_arena)) + } + + #[async_only] + pub(crate) fn next_tuple<'s, T: StorageTransaction + 'a, A: MetaArena + 'a>( + &'s mut self, + arena: &'s mut ExecContext<'_, 'a, '_, T>, + plan_arena: &'s mut ExecMetaArena, + ) -> std::pin::Pin> + Send + 's>> + { + Box::pin(self.next_tuple_inner(arena, plan_arena)) + } + + async fn reset_inner( + &mut self, + arena: &mut ExecContext<'_, 'a, '_, T>, + plan_arena: &mut ExecMetaArena, ) -> Result<(), DatabaseError> { match self { ExecNode::AddColumn(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::Analyze(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::ChangeColumn(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } #[cfg(feature = "copy")] ExecNode::CopyFromFile(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } #[cfg(feature = "copy")] ExecNode::CopyToFile(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::CreateIndex(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::CreateTable(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::CreateView(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::Delete(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::Describe(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::DropColumn(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::DropIndex(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::DropTable(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::DropView(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await + } + ExecNode::Dummy(exec) => { + >::reset(exec, arena, plan_arena).await } - ExecNode::Dummy(exec) => >::reset(exec, arena, plan_arena), ExecNode::Explain(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } #[cfg(feature = "spill")] ExecNode::ExternalSort(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::Filter(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::FunctionScan(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::HashAgg(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::HashJoin(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::IndexScan(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::Insert(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::Limit(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::MarkApply(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::IndexProbeJoin(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::NestedLoopJoin(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::Projection(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::RecursiveCte(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::RecursiveScan(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::ScalarQueryInit(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::ScalarSubquery(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::SetMembership(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::SeqScan(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::ShowTables(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::ShowViews(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::SimpleAgg(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::Sort(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::StreamAgg(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::StreamDistinct(exec) => { as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + .await } ExecNode::TopK(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::Truncate(exec) => { - >::reset(exec, arena, plan_arena) + >::reset(exec, arena, plan_arena).await } ExecNode::Union(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::Update(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::Values(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } ExecNode::Window(exec) => { - as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::reset(exec, arena, plan_arena).await } } } - pub(crate) fn next_tuple( + async fn next_tuple_inner( &mut self, arena: &mut ExecContext<'_, 'a, '_, T>, plan_arena: &mut ExecMetaArena, @@ -448,147 +499,156 @@ impl<'a> ExecNode<'a> { arena.buffer.result.status = None; match self { ExecNode::AddColumn(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::Analyze(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::ChangeColumn(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } #[cfg(feature = "copy")] ExecNode::CopyFromFile(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } #[cfg(feature = "copy")] ExecNode::CopyToFile(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::CreateIndex(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::CreateTable(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::CreateView(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::Delete(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::Describe(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::DropColumn(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::DropIndex(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::DropTable(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::DropView(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::Dummy(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::Explain(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } #[cfg(feature = "spill")] ExecNode::ExternalSort(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::Filter(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::FunctionScan(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::HashAgg(exec) => { as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + .await } ExecNode::HashJoin(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::IndexScan(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::Insert(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::Limit(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::MarkApply(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::IndexProbeJoin(exec) => { as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + .await } ExecNode::NestedLoopJoin(exec) => { as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + .await } ExecNode::Projection(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::RecursiveCte(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::RecursiveScan(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::ScalarQueryInit(exec) => { as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + .await } ExecNode::ScalarSubquery(exec) => { as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + .await } ExecNode::SetMembership(exec) => { as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + .await } ExecNode::SeqScan(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::ShowTables(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::ShowViews(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::SimpleAgg(exec) => { as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + .await } ExecNode::Sort(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::StreamAgg(exec) => { as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + .await } ExecNode::StreamDistinct(exec) => { as ExecutorNode<'a, T>>::next_tuple( exec, arena, plan_arena, ) + .await } ExecNode::TopK(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::Truncate(exec) => { - >::next_tuple(exec, arena, plan_arena) + >::next_tuple(exec, arena, plan_arena).await } ExecNode::Union(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::Update(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::Values(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } ExecNode::Window(exec) => { - as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena) + as ExecutorNode<'a, T>>::next_tuple(exec, arena, plan_arena).await } }?; match arena.buffer.result.status.unwrap_or(ExecStatus::End) { @@ -663,7 +723,7 @@ impl Drop for SpareIters { pub(crate) struct ExecArena<'a, 't, T: StorageTransaction + 'a> { table_cache: &'a TableCache, - transaction: *mut T, + transaction: TxnPtr, ddl_apply: Vec, recursive_input: Option, iters: Vec>, @@ -728,7 +788,7 @@ impl<'t, T: StorageTransaction + 't> StorageIter<'t, T> { } pub(crate) struct ExecArenaLocalState<'b, 'a, 't, T: StorageTransaction + 'a> { - transaction: *mut T, + transaction: TxnPtr, pub(crate) table_codec: &'b mut TableCodec, pub(crate) bound_buffers: &'b mut [Bytes; 2], pub(crate) result: &'b mut ExecResult, @@ -739,25 +799,25 @@ pub(crate) struct ExecArenaLocalState<'b, 'a, 't, T: StorageTransaction + 'a> { impl<'b, 'a, 't, T: StorageTransaction + 'a> ExecArenaLocalState<'b, 'a, 't, T> { pub(crate) fn transaction(&self) -> &'t T { - unsafe { &*self.transaction } + unsafe { &*self.transaction.0 } } pub(crate) fn transaction_codec_mut(&mut self) -> (&mut T, &mut TableCodec) { - unsafe { (&mut *self.transaction, &mut *self.table_codec) } + unsafe { (&mut *self.transaction.0, &mut *self.table_codec) } } pub(crate) fn tuple_transaction_codec_mut(&mut self) -> (&TupleImpl, &mut T, &mut TableCodec) { unsafe { ( &self.result.tuple, - &mut *self.transaction, + &mut *self.transaction.0, &mut *self.table_codec, ) } } pub(crate) fn transaction_codec(&mut self) -> (&'t T, &mut TableCodec) { - unsafe { (&*self.transaction, &mut *self.table_codec) } + unsafe { (&*self.transaction.0, &mut *self.table_codec) } } pub(crate) fn write_transaction_codec_ddl_apply_mut( @@ -765,7 +825,7 @@ impl<'b, 'a, 't, T: StorageTransaction + 'a> ExecArenaLocalState<'b, 'a, 't, T> ) -> (&mut T, &mut TableCodec, &mut Vec) { unsafe { ( - &mut *self.transaction, + &mut *self.transaction.0, &mut *self.table_codec, self.ddl_apply, ) @@ -808,7 +868,7 @@ impl<'a, 't, T: StorageTransaction + 'a> ExecArena<'a, 't, T> { pub(crate) fn new(table_cache: &'a TableCache, transaction: &'t T) -> Self { Self { table_cache, - transaction: transaction as *const T as *mut T, + transaction: TxnPtr(transaction as *const T as *mut T), ddl_apply: Vec::new(), recursive_input: None, iters: Vec::new(), @@ -857,7 +917,7 @@ impl<'c, 'a, 't, T: StorageTransaction + 'a> ExecContext<'c, 'a, 't, T> { pub(crate) fn transaction_codec_mut(&mut self) -> (&mut T, &mut TableCodec) { ( - unsafe { &mut *self.arena.transaction }, + unsafe { &mut *self.arena.transaction.0 }, &mut self.buffer.table_codec, ) } @@ -1029,6 +1089,56 @@ impl ExecBuilder for IterCounter { } } +/// A value that points into a bump arena owned by the same node, such as a `bumpalo::Vec`. +/// The node owns both and only reaches the value through `&mut self`, so the arena never has +/// another user to race with when the node moves to another thread. +pub(crate) struct ArenaBound(pub(crate) T); + +// SAFETY: see above. What is not `Send` here is only the pointer into the owned arena. +unsafe impl Send for ArenaBound {} + +impl std::ops::Deref for ArenaBound { + type Target = T; + + fn deref(&self) -> &T { + &self.0 + } +} + +impl std::ops::DerefMut for ArenaBound { + fn deref_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +/// The transaction an [`ExecArena`] lends to its nodes, both shared (iterators borrow it for +/// `'t`) and exclusive (writes). It stands for those references, so it may cross threads exactly +/// when they may: `&T` needs `T: Sync` and `&mut T` needs `T: Send`. +pub(crate) struct TxnPtr(*mut T); + +impl Clone for TxnPtr { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for TxnPtr {} + +impl TxnPtr { + pub(crate) fn new(ptr: *mut T) -> Self { + Self(ptr) + } + + pub(crate) fn get(self) -> *mut T { + self.0 + } +} + +// SAFETY: see above; the pointer is only dereferenced as `&T` / `&mut T`. +unsafe impl Send for TxnPtr {} +// SAFETY: as for `Send`. +unsafe impl Sync for TxnPtr {} + impl<'a, T: StorageTransaction + 'a> ExecBuilder for ExecContext<'_, 'a, '_, T> { fn alloc_iter(&mut self) -> IterSlot { self.arena.iters.push(StorageIter::Empty); @@ -1230,6 +1340,7 @@ pub(crate) fn build_write<'a, B: ExecBuilder>( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test_utils { use super::*; @@ -1241,9 +1352,13 @@ mod test_utils { } impl TestExecutor<'_, T> { - pub(crate) fn next_tuple(&mut self) -> Result, DatabaseError> { + pub(crate) async fn next_tuple(&mut self) -> Result, DatabaseError> { let mut context = ExecContext::new(&mut self.buffer, &mut self.arena); - if !self.root.next_tuple(&mut context, &mut self.plan_arena)? { + if !self + .root + .next_tuple(&mut context, &mut self.plan_arena) + .await? + { return Ok(None); } Ok(Some(self.buffer.result_mut())) @@ -1301,13 +1416,13 @@ mod test_utils { } } - pub fn try_collect( + pub async fn try_collect( executor: TestExecutor<'_, T>, ) -> Result, DatabaseError> { let mut executor = executor; let mut tuples = Vec::new(); - while let Some(tuple) = executor.next_tuple()? { + while let Some(tuple) = executor.next_tuple().await? { tuples.push(std::mem::take(tuple)); } Ok(tuples) @@ -1319,10 +1434,10 @@ mod test_utils { pub(crate) use test_utils::{execute_input, execute_input_mut, try_collect}; #[cfg(test)] +#[maybe_async] mod test { use super::*; use crate::planner::TableArenaCell; - use crate::storage::memory::MemoryStorage; use crate::storage::Storage; #[test] @@ -1335,7 +1450,7 @@ mod test { } #[test] - fn rewrite_projects_joined_row() -> Result<(), DatabaseError> { + async fn rewrite_projects_joined_row() -> Result<(), DatabaseError> { use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::types::LogicalType; @@ -1352,7 +1467,7 @@ mod test { let ints = |values: [i32; 2]| Tuple::new(None, values.map(DataValue::Int32).to_vec()); let tables = TableCache::default(); - let storage = MemoryStorage::new(); + let storage = crate::storage::new_test_storage().await; let transaction = storage.transaction()?; let mut exec_buffer = ExecBuffer::default(); let mut exec_arena = ExecArena::new(&tables, &transaction); diff --git a/src/execution/runs.rs b/src/execution/runs.rs index 4c1a8680..7b6c70a5 100644 --- a/src/execution/runs.rs +++ b/src/execution/runs.rs @@ -175,6 +175,7 @@ impl RunsCursor for &[DataValue] { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; diff --git a/src/execution/spill/codec.rs b/src/execution/spill/codec.rs index 0542e85a..2dafa2c4 100644 --- a/src/execution/spill/codec.rs +++ b/src/execution/spill/codec.rs @@ -391,6 +391,7 @@ fn estimated_dynamic_value_size(value: &DataValue) -> usize { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; use crate::types::value::Utf8Type; diff --git a/src/execution/spill/mod.rs b/src/execution/spill/mod.rs index b602244c..4a5ce054 100644 --- a/src/execution/spill/mod.rs +++ b/src/execution/spill/mod.rs @@ -70,7 +70,7 @@ pub(crate) enum FlushMode { } type OnFlush<'on_flush, T> = - Box) -> Result<(), DatabaseError> + 'on_flush>; + Box) -> Result<(), DatabaseError> + Send + 'on_flush>; enum ReadState { Memory(std::vec::IntoIter), @@ -111,7 +111,7 @@ impl<'on_flush, T: SpillCodec> SpillVec<'on_flush, T> { pub(crate) fn on_flush(mut self, on_flush: F) -> Self where - F: FnMut(FlushMode, &mut Vec) -> Result<(), DatabaseError> + 'on_flush, + F: FnMut(FlushMode, &mut Vec) -> Result<(), DatabaseError> + Send + 'on_flush, { if let Ok(state) = &mut self.writer { state.on_flush = Some(Box::new(on_flush)); @@ -453,6 +453,7 @@ fn create_spill_file() -> Result<(File, PathBuf), DatabaseError> { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; use crate::types::tuple::Tuple; diff --git a/src/execution/spill/runs.rs b/src/execution/spill/runs.rs index 2e6b8617..3b59dbab 100644 --- a/src/execution/spill/runs.rs +++ b/src/execution/spill/runs.rs @@ -210,6 +210,7 @@ impl RunsCursor for SpillRunsCursor<'_> { } #[cfg(test)] +#[maybe_async] mod tests { use super::*; diff --git a/src/expression/eq_col.rs b/src/expression/eq_col.rs index 32692595..00403c0b 100644 --- a/src/expression/eq_col.rs +++ b/src/expression/eq_col.rs @@ -540,6 +540,7 @@ impl ExprVisitor for EqIgnoreColRefPosVisitor<'_, A> { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/expression/evaluator.rs b/src/expression/evaluator.rs index ba10d710..123c136d 100644 --- a/src/expression/evaluator.rs +++ b/src/expression/evaluator.rs @@ -414,6 +414,7 @@ fn trim_string(value: &str, trim_what: &str, trim_where: Option) // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; use crate::expression::AliasType; diff --git a/src/expression/function/table.rs b/src/expression/function/table.rs index ff746410..891190c0 100644 --- a/src/expression/function/table.rs +++ b/src/expression/function/table.rs @@ -59,12 +59,15 @@ impl Hash for TableFunction { self.summary().hash(state); } } + +pub type TableFunctionIter = Box> + Send>; + pub trait TableFunctionImpl: Debug + Send + Sync { fn eval( &self, args: &[ExprRef], arena: &(dyn MetaArena + '_), - ) -> Result>>, DatabaseError>; + ) -> Result; fn summary(&self) -> &FunctionSummary; diff --git a/src/expression/mod.rs b/src/expression/mod.rs index 49e39a5c..c8120981 100644 --- a/src/expression/mod.rs +++ b/src/expression/mod.rs @@ -1088,6 +1088,7 @@ impl fmt::Display for UnaryOperator { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use crate::catalog::{ColumnCatalog, ColumnDesc}; use crate::db::test::build_table; @@ -1108,8 +1109,7 @@ mod test { use crate::planner::operator::sort::SortField; use crate::planner::{ExprRef, MetaArena, PlanArena, TableArenaCell}; use crate::serdes::{ReferenceDecodeContext, ReferenceSerialization}; - use crate::storage::rocksdb::RocksStorage; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::storage::{Storage, StorageTransaction}; use crate::types::evaluator::{binary_create, cast_create, unary_create}; use crate::types::value::{DataValue, Utf8Type}; @@ -1117,14 +1117,13 @@ mod test { use crate::types::LogicalType; use std::borrow::Cow; use std::io::{Cursor, Seek, SeekFrom}; - use tempfile::TempDir; #[test] - fn test_serialization() -> Result<(), DatabaseError> { + async fn test_serialization() -> Result<(), DatabaseError> { fn fn_assert( cursor: &mut Cursor>, expr: ScalarExpression, - drive: Option<&mut ReferenceDecodeContext<'_, RocksTransaction>>, + drive: Option<&mut ReferenceDecodeContext<'_, TestTransaction>>, arena: &mut (dyn MetaArena + '_), ) -> Result<(), DatabaseError> { let expr = arena.alloc_expression(expr); @@ -1141,8 +1140,7 @@ mod test { Ok(()) } - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + let storage = crate::storage::new_test_storage().await; let mut transaction = storage.transaction()?; let mut names = crate::storage::NameMapping::default(); let mut table_cache = crate::storage::TableCache::default(); @@ -1167,7 +1165,8 @@ mod test { &mut table_cache, &mut transaction, &mut plan_arena, - )?; + ) + .await?; let mut plan_arena = PlanArena::new(&table_arena); let mut cursor = Cursor::new(Vec::new()); diff --git a/src/expression/range_detacher.rs b/src/expression/range_detacher.rs index fd357d5a..ed41d712 100644 --- a/src/expression/range_detacher.rs +++ b/src/expression/range_detacher.rs @@ -1638,6 +1638,7 @@ impl fmt::Display for Range { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] pub(crate) mod test_support { use super::*; @@ -1672,6 +1673,7 @@ pub(crate) mod test_support { #[cfg(all(test, not(target_arch = "wasm32")))] #[allow(clippy::uninlined_format_args)] +#[maybe_async] mod test { use crate::binder::test::build_t1_table; use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef}; @@ -1903,8 +1905,8 @@ mod test { } #[test] - fn test_detach_consumes_and_predicates_with_residual() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_detach_consumes_and_predicates_with_residual() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut plan_arena = crate::planner::PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select * from t1 where c1 > 10 and c2 > 20", @@ -2036,8 +2038,9 @@ mod test { } #[test] - fn test_detach_consumes_complete_or_only_when_both_sides_match() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_detach_consumes_complete_or_only_when_both_sides_match( + ) -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut plan_arena = crate::planner::PlanArena::new(&table_state.table_arena); let plan = table_state .plan_with_arena("select * from t1 where c1 = 1 or c1 = 2", &mut plan_arena)?; @@ -2062,8 +2065,8 @@ mod test { } #[test] - fn test_detach_does_not_partially_consume_or() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_detach_does_not_partially_consume_or() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut plan_arena = crate::planner::PlanArena::new(&table_state.table_arena); let plan = table_state .plan_with_arena("select * from t1 where c1 = 1 or c2 = 2", &mut plan_arena)?; @@ -2080,8 +2083,8 @@ mod test { } #[test] - fn test_detach_ideal_cases() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_detach_ideal_cases() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut plan_arena = crate::planner::PlanArena::new(&table_state.table_arena); { let plan = @@ -2739,8 +2742,8 @@ mod test { } #[test] - fn test_detach_only_conjunction_can_keep_partial_range() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_detach_only_conjunction_can_keep_partial_range() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut plan_arena = crate::planner::PlanArena::new(&table_state.table_arena); let mut detach_c1 = |sql: &str| -> Result, DatabaseError> { let plan = table_state.plan_with_arena(sql, &mut plan_arena)?; @@ -2788,8 +2791,8 @@ mod test { // Tips: `null` should be First #[test] - fn test_detach_null_cases() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_detach_null_cases() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut plan_arena = crate::planner::PlanArena::new(&table_state.table_arena); // eq { @@ -3879,8 +3882,8 @@ mod test { /// AND over a union must intersect every piece instead of keeping the disjoint ones. #[test] - fn test_detach_and_over_or() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_detach_and_over_or() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut plan_arena = crate::planner::PlanArena::new(&table_state.table_arena); let eq = |v| Range::Eq(DataValue::Int32(v).into()); let cases = [ diff --git a/src/function/numbers.rs b/src/function/numbers.rs index 6613ad45..815ba1e5 100644 --- a/src/function/numbers.rs +++ b/src/function/numbers.rs @@ -15,7 +15,7 @@ use crate::catalog::ColumnCatalog; use crate::catalog::ColumnDesc; use crate::errors::DatabaseError; -use crate::expression::function::table::TableFunctionImpl; +use crate::expression::function::table::{TableFunctionImpl, TableFunctionIter}; use crate::expression::function::FunctionSummary; use crate::planner::MetaArena; use crate::planner::{ExprRef, TableArena}; @@ -49,7 +49,7 @@ impl TableFunctionImpl for Numbers { &self, args: &[ExprRef], arena: &(dyn MetaArena + '_), - ) -> Result>>, DatabaseError> { + ) -> Result { let mut value = arena .expression(args[0]) .eval::<[DataValue]>(arena, None)? @@ -62,7 +62,7 @@ impl TableFunctionImpl for Numbers { Ok( Box::new((0..num).map(|i| Ok(Tuple::new(None, vec![DataValue::Int32(i)])))) - as Box>>, + as TableFunctionIter, ) } diff --git a/src/lib.rs b/src/lib.rs index b8235145..0a440516 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -101,9 +101,19 @@ //! ``` #![allow(unused_doc_comments)] extern crate core; +// Puts `#[maybe_async]` in scope everywhere in the crate, test modules included. +#[macro_use] +extern crate kite_sql_serde_macros; #[cfg(all(feature = "spill", feature = "wasm", not(clippy)))] compile_error!("features `spill` and `wasm` are mutually exclusive"); +#[cfg(all(feature = "async", any(feature = "rocksdb", feature = "lmdb")))] +compile_error!("feature `async` is mutually exclusive with the sync storages `rocksdb` and `lmdb`"); +#[cfg(all( + feature = "async", + any(feature = "python", feature = "shell", feature = "wasm") +))] +compile_error!("features `python`, `shell` and `wasm` use the sync API and exclude `async`"); #[cfg(all(feature = "spill", target_arch = "wasm32"))] compile_error!("feature `spill` is not supported on wasm32 targets"); @@ -117,6 +127,7 @@ mod function; pub(crate) mod iter_ext; #[cfg(feature = "macros")] pub mod macros; +mod maybe_async; mod optimizer; #[cfg(feature = "orm")] pub mod orm; diff --git a/src/macros/mod.rs b/src/macros/mod.rs index ec32dbc3..1def9581 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -140,7 +140,7 @@ macro_rules! scala_function { /// .map(|i| Ok(Tuple::new(None, vec![ /// DataValue::Int32(Some(i)), /// DataValue::Int32(Some(i)), -/// ])))) as Box>>) +/// ])))) as TableFunctionIter) /// })); /// /// let mut kite_sql = DataBaseBuilder::path("./example").build()?; @@ -175,7 +175,7 @@ macro_rules! table_function { impl ::kite_sql::expression::function::table::TableFunctionImpl for $struct_name { #[allow(unused_variables, clippy::redundant_closure_call)] - fn eval(&self, args: &[::kite_sql::planner::ExprRef], arena: &dyn ::kite_sql::planner::MetaArena) -> Result>>, ::kite_sql::errors::DatabaseError> { + fn eval(&self, args: &[::kite_sql::planner::ExprRef], arena: &dyn ::kite_sql::planner::MetaArena) -> Result<::kite_sql::expression::function::table::TableFunctionIter, ::kite_sql::errors::DatabaseError> { let mut _index = 0; $closure($({ diff --git a/src/maybe_async.rs b/src/maybe_async.rs new file mode 100644 index 00000000..fd26bcf5 --- /dev/null +++ b/src/maybe_async.rs @@ -0,0 +1,57 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! One source, two paths: an item written once as `async` and marked `#[maybe_async]` is compiled as +//! the sync path by default and as the async path with the `async` feature, under the same +//! names. See `kite_sql_serde_macros::maybe_async` for the rules. +//! +//! `MaybeSend` / `MaybeSync` / `MaybeStatic` hold for every type on the sync path and mean +//! `Send` / `Sync` / `'static` on the async path. + +#[cfg(not(feature = "async"))] +mod bounds { + /// `Send` on the async path; implemented by every type on the sync path. + pub trait MaybeSend {} + impl MaybeSend for T {} + + /// `Sync` on the async path; implemented by every type on the sync path. + pub trait MaybeSync {} + impl MaybeSync for T {} + + /// `'static` on the async path; implemented by every type on the sync path. + pub trait MaybeStatic {} + impl MaybeStatic for T {} +} + +#[cfg(feature = "async")] +mod bounds { + pub use core::marker::{Send as MaybeSend, Sync as MaybeSync}; + + /// Async transactions must be `'static`: the compiler loses the implied `T: 't` bounds when + /// it checks that a future holding `&'t T` is `Send`, and `T: 'static` restores them for + /// every `'t`. + pub trait MaybeStatic: 'static {} + impl MaybeStatic for T {} +} + +pub use bounds::{MaybeSend, MaybeStatic, MaybeSync}; + +#[cfg(all(test, feature = "async"))] +pub(crate) fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to start a tokio runtime") + .block_on(future) +} diff --git a/src/optimizer/core/cm_sketch.rs b/src/optimizer/core/cm_sketch.rs index 3a911844..b007ccfc 100644 --- a/src/optimizer/core/cm_sketch.rs +++ b/src/optimizer/core/cm_sketch.rs @@ -393,6 +393,7 @@ impl ReferenceSerialization for CountMinSketch { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::expression::range_detacher::Range; use crate::optimizer::core::cm_sketch::{ @@ -400,7 +401,7 @@ mod tests { }; use crate::planner::TableArena; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::value::DataValue; use std::collections::Bound; use std::io::Cursor; @@ -555,7 +556,7 @@ mod tests { let mut bytes = Vec::new(); let mut arena = TableArena::default(); cms.encode(&mut bytes, false, &arena).unwrap(); - let decoded = CountMinSketch::::decode::( + let decoded = CountMinSketch::::decode::( &mut Cursor::new(bytes), None, &mut arena, diff --git a/src/optimizer/core/histogram.rs b/src/optimizer/core/histogram.rs index 8205c4ae..85f6ec7f 100644 --- a/src/optimizer/core/histogram.rs +++ b/src/optimizer/core/histogram.rs @@ -848,6 +848,7 @@ impl HistogramMeta { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::errors::DatabaseError; use crate::expression::range_detacher::{ComputedValue, Range, RangeValue}; diff --git a/src/optimizer/core/hll.rs b/src/optimizer/core/hll.rs index f9807604..b58ae949 100644 --- a/src/optimizer/core/hll.rs +++ b/src/optimizer/core/hll.rs @@ -98,6 +98,7 @@ impl HyperLogLog { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::HyperLogLog; diff --git a/src/optimizer/core/kll_sketch.rs b/src/optimizer/core/kll_sketch.rs index 4fa6a9ff..15331999 100644 --- a/src/optimizer/core/kll_sketch.rs +++ b/src/optimizer/core/kll_sketch.rs @@ -348,6 +348,7 @@ fn compare_values(left: &DataValue, right: &DataValue) -> Ordering { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::KllSketchBuilder; use crate::errors::DatabaseError; diff --git a/src/optimizer/core/statistics_meta.rs b/src/optimizer/core/statistics_meta.rs index ee05f545..a2b98204 100644 --- a/src/optimizer/core/statistics_meta.rs +++ b/src/optimizer/core/statistics_meta.rs @@ -156,6 +156,7 @@ impl IndexStatistics { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::errors::DatabaseError; use crate::expression::range_detacher::Range; diff --git a/src/optimizer/core/top_n.rs b/src/optimizer/core/top_n.rs index 6d9656ae..9b11d76d 100644 --- a/src/optimizer/core/top_n.rs +++ b/src/optimizer/core/top_n.rs @@ -225,6 +225,7 @@ impl ColumnTopN { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::ColumnTopN; use crate::types::value::DataValue; diff --git a/src/optimizer/heuristic/optimizer.rs b/src/optimizer/heuristic/optimizer.rs index a8c9a4a7..9feb2a8b 100644 --- a/src/optimizer/heuristic/optimizer.rs +++ b/src/optimizer/heuristic/optimizer.rs @@ -686,6 +686,7 @@ impl ImplementationRuleIndex { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::binder::{Binder, BinderContext}; use crate::db::DataBaseBuilder; @@ -824,18 +825,26 @@ mod tests { } #[test] - fn test_find_best_selects_cheapest_scan() -> Result<(), DatabaseError> { + async fn test_find_best_selects_cheapest_scan() -> Result<(), DatabaseError> { let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let mut database = DataBaseBuilder::path(temp_dir.path()).build_rocksdb()?; - database.ddl("create table t1 (c1 int primary key, c2 int)")?; - database.ddl("create table t2 (c3 int primary key, c4 int)")?; + let mut database = DataBaseBuilder::path(temp_dir.path()) + .build_in_memory() + .await?; + database + .ddl("create table t1 (c1 int primary key, c2 int)") + .await?; + database + .ddl("create table t2 (c3 int primary key, c4 int)") + .await?; for i in 0..1000 { database - .run(format!("insert into t1 values({}, {})", i, i + 1).as_str())? - .done()?; + .run(format!("insert into t1 values({}, {})", i, i + 1).as_str()) + .await? + .done() + .await?; } - database.analyze("t1")?; + database.analyze("t1").await?; let transaction = database.storage.transaction()?; let c1_column = transaction diff --git a/src/optimizer/plan_utils.rs b/src/optimizer/plan_utils.rs index 7a24554c..2496b3ec 100644 --- a/src/optimizer/plan_utils.rs +++ b/src/optimizer/plan_utils.rs @@ -270,6 +270,7 @@ pub(crate) fn scan_column( // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; diff --git a/src/optimizer/rule/implementation/dql/filter.rs b/src/optimizer/rule/implementation/dql/filter.rs index c69040ef..68c65603 100644 --- a/src/optimizer/rule/implementation/dql/filter.rs +++ b/src/optimizer/rule/implementation/dql/filter.rs @@ -219,6 +219,7 @@ pub(crate) fn estimate_predicate_rows( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/optimizer/rule/implementation/mod.rs b/src/optimizer/rule/implementation/mod.rs index 6e4e5501..d4c573e0 100644 --- a/src/optimizer/rule/implementation/mod.rs +++ b/src/optimizer/rule/implementation/mod.rs @@ -321,6 +321,7 @@ impl ImplementationRule for ImplementationRuleImpl { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; @@ -360,7 +361,7 @@ mod tests { } } - fn with_operator_from_sql( + async fn with_operator_from_sql( sql: &str, predicate: F, f: impl FnOnce(Operator, &PlanArena) -> Result, @@ -368,7 +369,7 @@ mod tests { where F: Fn(&Operator) -> bool + Copy, { - let tables = build_t1_table()?; + let tables = build_t1_table().await?; let mut arena = PlanArena::new(&tables.table_arena); let plan = tables.plan_with_arena(sql, &mut arena)?; let operator = find_operator(&plan, predicate).cloned().ok_or_else(|| { @@ -405,7 +406,7 @@ mod tests { .0) } - fn assert_sql_rule( + async fn assert_sql_rule( sql: &str, rule: ImplementationRuleImpl, predicate: impl Fn(&Operator) -> bool + Copy, @@ -413,32 +414,36 @@ mod tests { ) -> Result<(), DatabaseError> { let option = with_operator_from_sql(sql, predicate, |operator, arena| { best_option(rule, &operator, arena) - })?; + }) + .await?; assert_eq!(option.plan, expected_plan); Ok(()) } #[test] - fn test_single_mapping_implementations() -> Result<(), DatabaseError> { + async fn test_single_mapping_implementations() -> Result<(), DatabaseError> { assert_sql_rule( "select count(c1) from t1", ImplementationRuleImpl::SimpleAggregate, |op| matches!(op, Operator::Aggregate(agg) if agg.groupby_exprs.is_empty()), PlanImpl::SimpleAggregate, - )?; + ) + .await?; assert_sql_rule( "select c1, count(c2) from t1 group by c1", ImplementationRuleImpl::GroupByAggregate, |op| matches!(op, Operator::Aggregate(agg) if !agg.groupby_exprs.is_empty()), PlanImpl::HashAggregate, - )?; + ) + .await?; assert_sql_rule( "select 1", ImplementationRuleImpl::Dummy, |op| matches!(op, Operator::Dummy), PlanImpl::Dummy, - )?; + ) + .await?; with_operator_from_sql( "select * from t1 where c1 = 1", |op| matches!(op, Operator::Filter(_)), @@ -448,13 +453,15 @@ mod tests { assert_eq!(option.row_count, RowCountOption::Follow); Ok(()) }, - )?; + ) + .await?; assert_sql_rule( "select * from t1 limit 1", ImplementationRuleImpl::Limit, |op| matches!(op, Operator::Limit(_)), PlanImpl::Limit, - )?; + ) + .await?; with_operator_from_sql( "select * from t1 where exists(select * from t2)", |op| matches!(op, Operator::MarkApply(_)), @@ -464,61 +471,70 @@ mod tests { assert_eq!(option.row_count, RowCountOption::Follow); Ok(()) }, - )?; + ) + .await?; assert_sql_rule( "select c1 from t1", ImplementationRuleImpl::Projection, |op| matches!(op, Operator::Project(_)), PlanImpl::Project, - )?; + ) + .await?; assert_sql_rule( "select (select c3 from t2 limit 1) from t1", ImplementationRuleImpl::ScalarQueryInit, |op| matches!(op, Operator::ScalarQueryInit(_)), PlanImpl::ScalarQueryInit, - )?; + ) + .await?; assert_sql_rule( "select (select c3 from t2 limit 1) from t1", ImplementationRuleImpl::ScalarSubquery, |op| matches!(op, Operator::ScalarSubquery(_)), PlanImpl::ScalarSubquery, - )?; + ) + .await?; assert_sql_rule( "values (1, 2)", ImplementationRuleImpl::Values, |op| matches!(op, Operator::Values(_)), PlanImpl::Values, - )?; + ) + .await?; assert_sql_rule( "select row_number() over (order by c1) from t1", ImplementationRuleImpl::Window, |op| matches!(op, Operator::Window(_)), PlanImpl::Window, - )?; + ) + .await?; assert_sql_rule( "select * from t1", ImplementationRuleImpl::SeqScan, |op| matches!(op, Operator::TableScan(_)), PlanImpl::SeqScan, - )?; + ) + .await?; Ok(()) } #[test] - fn test_join_sort_topk_and_function_scan_implementations() -> Result<(), DatabaseError> { + async fn test_join_sort_topk_and_function_scan_implementations() -> Result<(), DatabaseError> { assert_sql_rule( "select * from t1 join t2 on c1 = c3", ImplementationRuleImpl::HashJoin, |op| matches!(op, Operator::Join(_)), PlanImpl::HashJoin, - )?; + ) + .await?; assert_sql_rule( "select * from t1 cross join t2", ImplementationRuleImpl::HashJoin, |op| matches!(op, Operator::Join(_)), PlanImpl::NestLoopJoin, - )?; + ) + .await?; with_operator_from_sql( "select * from t1 order by c1", @@ -554,7 +570,8 @@ mod tests { Ok(()) }, - )?; + ) + .await?; let table_arena = TableArenaCell::default(); let numbers = Numbers::new(); let mut schema = Vec::new(); @@ -580,81 +597,93 @@ mod tests { } #[test] - fn test_dml_and_ddl_implementations() -> Result<(), DatabaseError> { + async fn test_dml_and_ddl_implementations() -> Result<(), DatabaseError> { assert_sql_rule( "analyze table t1", ImplementationRuleImpl::Analyze, |op| matches!(op, Operator::Analyze(_)), PlanImpl::Analyze, - )?; + ) + .await?; assert_sql_rule( "delete from t1 where c1 = 1", ImplementationRuleImpl::Delete, |op| matches!(op, Operator::Delete(_)), PlanImpl::Delete, - )?; + ) + .await?; assert_sql_rule( "insert into t1 values (1, 2)", ImplementationRuleImpl::Insert, |op| matches!(op, Operator::Insert(_)), PlanImpl::Insert, - )?; + ) + .await?; assert_sql_rule( "update t1 set c2 = 3 where c1 = 1", ImplementationRuleImpl::Update, |op| matches!(op, Operator::Update(_)), PlanImpl::Update, - )?; + ) + .await?; #[cfg(feature = "copy")] assert_sql_rule( "copy t1 from 'in.csv'", ImplementationRuleImpl::CopyFromFile, |op| matches!(op, Operator::CopyFromFile(_)), PlanImpl::CopyFromFile, - )?; + ) + .await?; #[cfg(feature = "copy")] assert_sql_rule( "copy t1 to 'out.csv'", ImplementationRuleImpl::CopyToFile, |op| matches!(op, Operator::CopyToFile(_)), PlanImpl::CopyToFile, - )?; + ) + .await?; assert_sql_rule( "alter table t1 add column c5 int null", ImplementationRuleImpl::AddColumn, |op| matches!(op, Operator::AddColumn(_)), PlanImpl::AddColumn, - )?; + ) + .await?; assert_sql_rule( "alter table t1 alter column c2 type bigint", ImplementationRuleImpl::ChangeColumn, |op| matches!(op, Operator::ChangeColumn(_)), PlanImpl::ChangeColumn, - )?; + ) + .await?; assert_sql_rule( "alter table t1 drop column c2", ImplementationRuleImpl::DropColumn, |op| matches!(op, Operator::DropColumn(_)), PlanImpl::DropColumn, - )?; + ) + .await?; assert_sql_rule( "create table t3 (id int primary key)", ImplementationRuleImpl::CreateTable, |op| matches!(op, Operator::CreateTable(_)), PlanImpl::CreateTable, - )?; + ) + .await?; assert_sql_rule( "drop table t1", ImplementationRuleImpl::DropTable, |op| matches!(op, Operator::DropTable(_)), PlanImpl::DropTable, - )?; + ) + .await?; assert_sql_rule( "truncate table t1", ImplementationRuleImpl::Truncate, |op| matches!(op, Operator::Truncate(_)), PlanImpl::Truncate, - )?; + ) + .await?; Ok(()) } diff --git a/src/optimizer/rule/normalization/column_pruning.rs b/src/optimizer/rule/normalization/column_pruning.rs index b85c21b6..f98ae1c3 100644 --- a/src/optimizer/rule/normalization/column_pruning.rs +++ b/src/optimizer/rule/normalization/column_pruning.rs @@ -915,6 +915,7 @@ impl NormalizationRule for SkipResidualFilterColumns { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; @@ -993,8 +994,8 @@ mod tests { } #[test] - fn test_column_pruning_project_single_side() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_column_pruning_project_single_side() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let arena = PlanArena::new(&table_state.table_arena); let mut arena = arena; let best_plan = optimize_column_pruning(&table_state, &mut arena, "select c1 from t1")?; @@ -1009,8 +1010,8 @@ mod tests { } #[test] - fn test_column_pruning_filter_single_side() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_column_pruning_filter_single_side() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let arena = PlanArena::new(&table_state.table_arena); let mut arena = arena; let best_plan = @@ -1026,8 +1027,8 @@ mod tests { } #[test] - fn test_column_pruning_aggregate_single_side() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_column_pruning_aggregate_single_side() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let arena = PlanArena::new(&table_state.table_arena); let mut arena = arena; let best_plan = @@ -1043,8 +1044,8 @@ mod tests { } #[test] - fn test_column_pruning_sort_single_side() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_column_pruning_sort_single_side() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let arena = PlanArena::new(&table_state.table_arena); let mut arena = arena; let best_plan = @@ -1060,8 +1061,8 @@ mod tests { } #[test] - fn test_column_pruning_limit_single_side() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_column_pruning_limit_single_side() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let arena = PlanArena::new(&table_state.table_arena); let mut arena = arena; let best_plan = @@ -1077,8 +1078,8 @@ mod tests { } #[test] - fn test_column_pruning() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_column_pruning() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state .plan_with_arena("select c1, c3 from t1 left join t2 on c1 = c3", &mut arena)?; @@ -1128,8 +1129,9 @@ mod tests { } #[test] - fn test_column_pruning_keeps_join_filter_columns_on_both_sides() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_column_pruning_keeps_join_filter_columns_on_both_sides( + ) -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let best_plan = optimize_column_pruning( &table_state, @@ -1144,8 +1146,8 @@ mod tests { } #[test] - fn test_column_pruning_keeps_mark_apply_predicate_columns() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_column_pruning_keeps_mark_apply_predicate_columns() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let best_plan = optimize_column_pruning( &table_state, @@ -1164,8 +1166,8 @@ mod tests { } #[test] - fn test_column_pruning_preserves_union_input_mapping() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_column_pruning_preserves_union_input_mapping() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let best_plan = optimize_column_pruning( &table_state, @@ -1184,7 +1186,7 @@ mod tests { } #[test] - fn skip_residual_filter_columns_in_index_reads() -> Result<(), DatabaseError> { + async fn skip_residual_filter_columns_in_index_reads() -> Result<(), DatabaseError> { use crate::db::DataBaseBuilder; use crate::planner::operator::{PhysicalOption, PlanImpl}; use crate::types::serialize::TupleValueSerializableImpl; @@ -1199,17 +1201,23 @@ mod tests { let mut db = DataBaseBuilder::path(".") .histogram_buckets(2) - .build_in_memory()?; - db.ddl("create table skip_t(id int primary key, a int, b int, c int)")?; - db.ddl("create index skip_ab on skip_t(a, b)")?; - db.run("insert into skip_t values(1, 1, 10, 100), (2, 1, 20, 200), (3, 2, 30, 300)")? - .done()?; + .build_in_memory() + .await?; + db.ddl("create table skip_t(id int primary key, a int, b int, c int)") + .await?; + db.ddl("create index skip_ab on skip_t(a, b)").await?; + db.run("insert into skip_t values(1, 1, 10, 100), (2, 1, 20, 200), (3, 2, 30, 300)") + .await? + .done() + .await?; // Rows of other `a`s, so that `a = 1` is selective enough for the index. for id in 4..64 { - db.run(format!("insert into skip_t values({id}, {id}, 0, 0)"))? - .done()?; + db.run(format!("insert into skip_t values({id}, {id}, 0, 0)")) + .await? + .done() + .await?; } - db.analyze("skip_t")?; + db.analyze("skip_t").await?; let mut tx = db.new_transaction(false)?; for (sql, mapping, expected) in [ @@ -1246,13 +1254,14 @@ mod tests { assert!(info.covered, "{sql}"); assert_eq!(info.cover_mapping, mapping, "{sql}"); - let mut iter = program.execute(&mut tx, [])?; + let mut iter = program.execute(&mut tx, []).await?; let mut rows = Vec::new(); while iter - .next_tuple(|_, row| rows.push(row.value_at(0).clone()))? + .next_tuple(|_, row| rows.push(row.value_at(0).clone())) + .await? .is_some() {} - iter.done()?; + iter.done().await?; let expected: Vec<_> = expected.into_iter().map(DataValue::Int32).collect(); assert_eq!(rows, expected, "{sql}"); } @@ -1283,27 +1292,34 @@ mod tests { }) .collect::>(); assert_eq!(skipped, vec![true, false]); - let mut iter = program.execute(&mut tx, [])?; + let mut iter = program.execute(&mut tx, []).await?; let mut rows = Vec::new(); while iter - .next_tuple(|_, row| rows.push(row.value_at(0).clone()))? + .next_tuple(|_, row| rows.push(row.value_at(0).clone())) + .await? .is_some() {} - iter.done()?; + iter.done().await?; assert_eq!(rows, vec![DataValue::Int32(200)]); // An update writes back the whole row it scans, so its scan stays pinned. drop(tx); - db.run("update skip_t set b = b + 1 where a = 1")?.done()?; - let mut iter = db.run("select id, a, b from skip_t where id <= 3 order by id")?; + db.run("update skip_t set b = b + 1 where a = 1") + .await? + .done() + .await?; + let mut iter = db + .run("select id, a, b from skip_t where id <= 3 order by id") + .await?; let mut rows = Vec::new(); while iter .next_tuple(|_, row| { rows.push((0..3).map(|i| row.value_at(i).clone()).collect::>()) - })? + }) + .await? .is_some() {} - iter.done()?; + iter.done().await?; let expected: Vec> = [[1, 1, 11], [2, 1, 21], [3, 2, 30]] .into_iter() .map(|row| row.into_iter().map(DataValue::Int32).collect()) diff --git a/src/optimizer/rule/normalization/combine_operators.rs b/src/optimizer/rule/normalization/combine_operators.rs index 12a91bd3..c70b43b1 100644 --- a/src/optimizer/rule/normalization/combine_operators.rs +++ b/src/optimizer/rule/normalization/combine_operators.rs @@ -129,6 +129,7 @@ impl NormalizationRule for CollapseGroupByAgg { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::binder::test::build_t1_table; use crate::catalog::ColumnCatalog; @@ -149,8 +150,8 @@ mod tests { } #[test] - fn test_combine_filter_keeps_reordered_projection() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_combine_filter_keeps_reordered_projection() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena("select * from t1 c where c.c1 > 1", &mut arena)?; let mut filter = plan.childrens.pop_only(); @@ -168,8 +169,8 @@ mod tests { } #[test] - fn test_combine_filter() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_combine_filter() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select * from (select * from t1 where c1 > 1) t where 1 = 1", @@ -206,8 +207,8 @@ mod tests { } #[test] - fn test_collapse_group_by_agg() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_collapse_group_by_agg() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state .plan_with_arena("select distinct c1, c2 from t1 group by c1, c2", &mut arena)?; diff --git a/src/optimizer/rule/normalization/compilation_in_advance.rs b/src/optimizer/rule/normalization/compilation_in_advance.rs index 310de4b8..5a16939d 100644 --- a/src/optimizer/rule/normalization/compilation_in_advance.rs +++ b/src/optimizer/rule/normalization/compilation_in_advance.rs @@ -78,6 +78,7 @@ impl NormalizationRule for EvaluatorBind { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/optimizer/rule/normalization/constant_propagation.rs b/src/optimizer/rule/normalization/constant_propagation.rs index b8da8838..631d07a7 100644 --- a/src/optimizer/rule/normalization/constant_propagation.rs +++ b/src/optimizer/rule/normalization/constant_propagation.rs @@ -356,6 +356,7 @@ impl NormalizationRule for PropagateJoinConstants { } #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::{eq_operands, for_each_conjunct, table_column}; use super::{PropagateFilterConstants, PropagateJoinConstants}; @@ -426,8 +427,8 @@ mod tests { } /// Runs the rules of the default "Predicate Pushdown" batch (same order as `db.rs`). - fn propagate_join(sql: &str) -> Result<(Vec, Vec), DatabaseError> { - let table_state = build_t1_table()?; + async fn propagate_join(sql: &str) -> Result<(Vec, Vec), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena(sql, &mut arena)?; let mut plan = HepOptimizerPipeline::builder() @@ -453,8 +454,8 @@ mod tests { } #[test] - fn filter_derives_constant_through_column_equality() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn filter_derives_constant_through_column_equality() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let mut plan = table_state.plan_with_arena("select * from t1 where c1 = c2 and c2 = 1", &mut arena)?; @@ -468,8 +469,8 @@ mod tests { } #[test] - fn filter_closes_equality_chains() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn filter_closes_equality_chains() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let mut plan = table_state.plan_with_arena( "select * from t1, t2 where c1 = c3 and c3 = c2 and c2 = 1", @@ -483,8 +484,8 @@ mod tests { } #[test] - fn filter_ignores_null_and_non_equality() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn filter_ignores_null_and_non_equality() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; for sql in [ "select * from t1 where c1 = c2 and c2 = null", "select * from t1 where c1 > c2 and c2 = 1", @@ -502,59 +503,59 @@ mod tests { } #[test] - fn inner_join_propagates_both_ways() -> Result<(), DatabaseError> { + async fn inner_join_propagates_both_ways() -> Result<(), DatabaseError> { assert_eq!( - propagate_join("select * from t1 join t2 on c1 = c3 where c1 = 1")?, + propagate_join("select * from t1 join t2 on c1 = c3 where c1 = 1").await?, (vec!["c1 = 1".into()], vec!["c3 = 1".into()]) ); assert_eq!( - propagate_join("select * from t1 join t2 on c1 = c3 where c3 = 1")?, + propagate_join("select * from t1 join t2 on c1 = c3 where c3 = 1").await?, (vec!["c1 = 1".into()], vec!["c3 = 1".into()]) ); Ok(()) } #[test] - fn left_join_only_constrains_right_side() -> Result<(), DatabaseError> { + async fn left_join_only_constrains_right_side() -> Result<(), DatabaseError> { assert_eq!( - propagate_join("select * from t1 left join t2 on c1 = c3 where c1 = 1")?, + propagate_join("select * from t1 left join t2 on c1 = c3 where c1 = 1").await?, (vec!["c1 = 1".into()], vec!["c3 = 1".into()]) ); // The non-preserved side's ON constant is pushed down, never derived onto the left. assert_eq!( - propagate_join("select * from t1 left join t2 on c1 = c3 and c3 = 1")?, + propagate_join("select * from t1 left join t2 on c1 = c3 and c3 = 1").await?, (vec![], vec!["c3 = 1".into()]) ); // The preserved side's ON constant stays on the join but still derives the right side. assert_eq!( - propagate_join("select * from t1 left join t2 on c1 = c3 and c1 = 1")?, + propagate_join("select * from t1 left join t2 on c1 = c3 and c1 = 1").await?, (vec![], vec!["c3 = 1".into()]) ); Ok(()) } #[test] - fn right_join_only_constrains_left_side() -> Result<(), DatabaseError> { + async fn right_join_only_constrains_left_side() -> Result<(), DatabaseError> { assert_eq!( - propagate_join("select * from t1 right join t2 on c1 = c3 where c3 = 1")?, + propagate_join("select * from t1 right join t2 on c1 = c3 where c3 = 1").await?, (vec!["c1 = 1".into()], vec!["c3 = 1".into()]) ); assert_eq!( - propagate_join("select * from t1 right join t2 on c1 = c3 and c1 = 1")?, + propagate_join("select * from t1 right join t2 on c1 = c3 and c1 = 1").await?, (vec!["c1 = 1".into()], vec![]) ); assert_eq!( - propagate_join("select * from t1 right join t2 on c1 = c3 and c3 = 1")?, + propagate_join("select * from t1 right join t2 on c1 = c3 and c3 = 1").await?, (vec!["c1 = 1".into()], vec![]) ); Ok(()) } - fn propagate_from_side( + async fn propagate_from_side( join: &str, fact_on_left: bool, ) -> Result<(bool, Vec, Vec), DatabaseError> { - let table_state = build_t1_table()?; + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let mut plan = table_state .plan_with_arena(format!("select * from t1 {join} t2 on c1 = c3"), &mut arena)?; @@ -586,7 +587,7 @@ mod tests { } #[test] - fn join_type_limits_propagation_direction() -> Result<(), DatabaseError> { + async fn join_type_limits_propagation_direction() -> Result<(), DatabaseError> { let one = |column: &str| vec![format!("{column} = 1")]; // (join, fact on left, expected (applied, left facts, right facts)) let cases = [ @@ -602,7 +603,7 @@ mod tests { ]; for (join, fact_on_left, expected) in cases { assert_eq!( - propagate_from_side(join, fact_on_left)?, + propagate_from_side(join, fact_on_left).await?, expected, "{join}, fact on left: {fact_on_left}" ); diff --git a/src/optimizer/rule/normalization/elimination.rs b/src/optimizer/rule/normalization/elimination.rs index d18263ff..517ac646 100644 --- a/src/optimizer/rule/normalization/elimination.rs +++ b/src/optimizer/rule/normalization/elimination.rs @@ -838,6 +838,7 @@ pub(crate) fn covers( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::{ EliminateIndexFilter, EliminateProject, EliminateRedundantSort, ForceSpillAggregate, @@ -927,8 +928,9 @@ mod tests { } #[test] - fn substitutes_through_filter_and_sort_without_cloning_sources() -> Result<(), DatabaseError> { - let table = build_t1_table()?; + async fn substitutes_through_filter_and_sort_without_cloning_sources( + ) -> Result<(), DatabaseError> { + let table = build_t1_table().await?; let mut arena = PlanArena::new(&table.table_arena); let scan = table.plan_with_arena("select c1, c2 from t1", &mut arena)?; let (input, replacement) = source(&mut arena, scan); @@ -995,8 +997,9 @@ mod tests { } #[test] - fn aggregate_preserves_input_project_and_separate_output_slots() -> Result<(), DatabaseError> { - let table = build_t1_table()?; + async fn aggregate_preserves_input_project_and_separate_output_slots( + ) -> Result<(), DatabaseError> { + let table = build_t1_table().await?; let mut arena = PlanArena::new(&table.table_arena); let scan = table.plan_with_arena("select c1, c2 from t1", &mut arena)?; let (input, replacement) = source(&mut arena, scan); @@ -1054,9 +1057,9 @@ mod tests { } #[test] - fn mapping_transfer_preserves_shared_source_expressions_across_plan_nodes( + async fn mapping_transfer_preserves_shared_source_expressions_across_plan_nodes( ) -> Result<(), DatabaseError> { - let table = build_t1_table()?; + let table = build_t1_table().await?; let mut arena = PlanArena::new(&table.table_arena); let scan = table.plan_with_arena("select c1, c2 from t1", &mut arena)?; let (input, replacement) = source(&mut arena, scan); @@ -1229,9 +1232,9 @@ mod tests { } #[test] - fn cross_join_inlines_computed_right_output_without_changing_source( + async fn cross_join_inlines_computed_right_output_without_changing_source( ) -> Result<(), DatabaseError> { - let table = build_t1_table()?; + let table = build_t1_table().await?; let mut arena = PlanArena::new(&table.table_arena); let scan = table.plan_with_arena("select c1, c2 from t1", &mut arena)?; let (right, original) = source(&mut arena, scan); @@ -1258,8 +1261,8 @@ mod tests { } #[test] - fn test_collapse_project() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_collapse_project() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena("select c1 from (select c1, c2 from t1) t", &mut arena)?; diff --git a/src/optimizer/rule/normalization/guard_mark_apply.rs b/src/optimizer/rule/normalization/guard_mark_apply.rs index 65bb0c3b..ce426c97 100644 --- a/src/optimizer/rule/normalization/guard_mark_apply.rs +++ b/src/optimizer/rule/normalization/guard_mark_apply.rs @@ -439,6 +439,7 @@ impl ExprVisitor> for SimpleGuardVisitor { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/optimizer/rule/normalization/join_reorder.rs b/src/optimizer/rule/normalization/join_reorder.rs index fa0d8ef2..57851388 100644 --- a/src/optimizer/rule/normalization/join_reorder.rs +++ b/src/optimizer/rule/normalization/join_reorder.rs @@ -716,6 +716,7 @@ impl NormalizationRule for GreedyJoinReorder { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/optimizer/rule/normalization/materialize_mark_apply.rs b/src/optimizer/rule/normalization/materialize_mark_apply.rs index a49936e3..019d0ad3 100644 --- a/src/optimizer/rule/normalization/materialize_mark_apply.rs +++ b/src/optimizer/rule/normalization/materialize_mark_apply.rs @@ -69,6 +69,7 @@ impl NormalizationRule for MaterializeMarkApply { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/optimizer/rule/normalization/min_max_top_k.rs b/src/optimizer/rule/normalization/min_max_top_k.rs index bbf062e1..370163e3 100644 --- a/src/optimizer/rule/normalization/min_max_top_k.rs +++ b/src/optimizer/rule/normalization/min_max_top_k.rs @@ -84,6 +84,7 @@ impl NormalizationRule for MinMaxToTopK { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::MinMaxToTopK; use crate::binder::test::build_t1_table; @@ -115,8 +116,8 @@ mod tests { } #[test] - fn test_min_to_topk() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_min_to_topk() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let mut plan = table_state.plan_with_arena("select min(c1) from t1", &mut arena)?; @@ -153,8 +154,8 @@ mod tests { } #[test] - fn test_max_to_topk() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_max_to_topk() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let mut plan = table_state.plan_with_arena("select max(c2) from t1", &mut arena)?; diff --git a/src/optimizer/rule/normalization/parameterized_index.rs b/src/optimizer/rule/normalization/parameterized_index.rs index 58c7927c..af59be6e 100644 --- a/src/optimizer/rule/normalization/parameterized_index.rs +++ b/src/optimizer/rule/normalization/parameterized_index.rs @@ -602,6 +602,7 @@ impl NormalizationRule for ParameterizeInnerJoin { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; @@ -748,6 +749,7 @@ mod tests { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod inner_join_tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/optimizer/rule/normalization/pushdown_limit.rs b/src/optimizer/rule/normalization/pushdown_limit.rs index a665cf3a..e34f001a 100644 --- a/src/optimizer/rule/normalization/pushdown_limit.rs +++ b/src/optimizer/rule/normalization/pushdown_limit.rs @@ -139,6 +139,7 @@ impl NormalizationRule for PushLimitIntoScan { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::binder::test::build_t1_table; use crate::errors::DatabaseError; @@ -149,8 +150,8 @@ mod tests { use crate::planner::PlanArena; #[test] - fn test_limit_project_transpose() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_limit_project_transpose() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena("select c1, c2 from t1 limit 1", &mut arena)?; @@ -178,8 +179,8 @@ mod tests { } #[test] - fn test_push_limit_through_join() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_push_limit_through_join() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select * from t1 left join t2 on c1 = c3 limit 1", @@ -215,8 +216,8 @@ mod tests { } #[test] - fn test_push_limit_into_table_scan() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_push_limit_into_table_scan() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena("select * from t1 limit 1 offset 1", &mut arena)?; diff --git a/src/optimizer/rule/normalization/pushdown_predicates.rs b/src/optimizer/rule/normalization/pushdown_predicates.rs index f8de79f5..c932a270 100644 --- a/src/optimizer/rule/normalization/pushdown_predicates.rs +++ b/src/optimizer/rule/normalization/pushdown_predicates.rs @@ -605,6 +605,7 @@ impl NormalizationRule for PushJoinPredicateIntoScan { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::binder::test::build_t1_table; @@ -679,12 +680,12 @@ mod tests { } #[test] - fn single_pass_filter_classification_preserves_reference_semantics() -> Result<(), DatabaseError> - { + async fn single_pass_filter_classification_preserves_reference_semantics( + ) -> Result<(), DatabaseError> { use crate::expression::AliasType; use crate::planner::operator::values::ValuesOperator; - let table_state = build_t1_table()?; + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let left_column = build_test_column(&mut arena, 1, "left")?; let right_column = build_test_column(&mut arena, 2, "right")?; @@ -737,8 +738,8 @@ mod tests { } #[test] - fn test_push_predicate_into_scan() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_push_predicate_into_scan() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); // 1 - c2 < 0 => c2 > 1, directly and through scalar initialization. for (sql, through_init) in [ @@ -788,8 +789,8 @@ mod tests { } #[test] - fn test_unique_composite_index_uses_tuple_ranges() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_unique_composite_index_uses_tuple_ranges() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let c1 = build_test_column(&mut arena, 1, "c1")?; let c2 = build_test_column(&mut arena, 2, "c2")?; @@ -1151,8 +1152,8 @@ mod tests { } #[test] - fn test_push_predicate_through_join_in_left_join() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_push_predicate_through_join_in_left_join() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select * from t1 left join t2 on c1 = c3 where c1 > 1 and c3 < 2", @@ -1201,8 +1202,8 @@ mod tests { } #[test] - fn test_push_predicate_through_join_in_right_join() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_push_predicate_through_join_in_right_join() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select * from t1 right join t2 on c1 = c3 where c1 > 1 and c3 < 2", @@ -1251,8 +1252,8 @@ mod tests { } #[test] - fn cross_join_without_conjunctive_keys_stays_cross() -> Result<(), DatabaseError> { - let tables = build_t1_table()?; + async fn cross_join_without_conjunctive_keys_stays_cross() -> Result<(), DatabaseError> { + let tables = build_t1_table().await?; for predicate in ["c1 = c3 or c2 = c4", "c1 < c3", "c1 > 0 and c3 < 9"] { let mut arena = PlanArena::new(&tables.table_arena); let plan = tables.plan_with_arena( @@ -1284,8 +1285,8 @@ mod tests { } #[test] - fn test_push_predicate_through_join_in_inner_join() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_push_predicate_through_join_in_inner_join() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select * from t1 inner join t2 on c1 = c3 where c1 > 1 and c3 < 2", @@ -1339,8 +1340,8 @@ mod tests { } #[test] - fn test_push_join_predicate_into_scan_inner_join() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_push_join_predicate_into_scan_inner_join() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select * from t1 inner join t2 on t1.c1 = t2.c3 and t1.c1 > 1 and t2.c3 < 2", @@ -1415,8 +1416,8 @@ mod tests { } #[test] - fn test_push_join_predicate_left_outer_preserve_left() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_push_join_predicate_left_outer_preserve_left() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select * from t1 left join t2 on t1.c1 = t2.c3 and t1.c1 > 1", @@ -1463,8 +1464,8 @@ mod tests { } #[test] - fn test_push_join_predicate_left_outer_push_right() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_push_join_predicate_left_outer_push_right() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select * from t1 left join t2 on t1.c1 = t2.c3 and t2.c3 < 2", diff --git a/src/optimizer/rule/normalization/simplification.rs b/src/optimizer/rule/normalization/simplification.rs index c5c3de4f..af99bd96 100644 --- a/src/optimizer/rule/normalization/simplification.rs +++ b/src/optimizer/rule/normalization/simplification.rs @@ -115,6 +115,7 @@ impl NormalizationRule for SimplifyFilter { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use crate::binder::test::build_t1_table; @@ -147,8 +148,8 @@ mod test { } #[test] - fn test_constant_calculation_omitted() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_constant_calculation_omitted() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); // (2 + (-1)) < -(c1 + 1) let plan = table_state.plan_with_arena( @@ -206,8 +207,8 @@ mod test { } #[test] - fn test_constant_cast_elimination() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_constant_cast_elimination() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena( "select cast(1 as int), cast(2 as int) + 1 from t1 where cast(3 as int) = 3", @@ -279,8 +280,8 @@ mod test { } #[test] - fn test_simplify_filter_single_column() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_single_column() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); // c1 + 1 < -1 => c1 < -2 let plan_1 = @@ -357,8 +358,8 @@ mod test { } #[test] - fn test_simplify_filter_boolean_wrapped_range_comparison() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_boolean_wrapped_range_comparison() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let expected = Some(Range::Scope { min: Bound::Unbounded, @@ -381,8 +382,8 @@ mod test { } #[test] - fn test_simplify_filter_repeating_column() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_repeating_column() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena("select * from t1 where -(c1 + 1) > c2", &mut arena)?; @@ -435,8 +436,9 @@ mod test { } #[test] - fn test_non_comparison_does_not_apply_pending_rearrangement() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_non_comparison_does_not_apply_pending_rearrangement() -> Result<(), DatabaseError> + { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan = table_state.plan_with_arena("select * from t1 where (c1 + 1) % 2 = 0", &mut arena)?; @@ -502,8 +504,8 @@ mod test { } #[test] - fn test_simplify_filter_multiple_column() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_multiple_column() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); // c1 + 1 < -1 => c1 < -2 let plan_1 = table_state.plan_with_arena( @@ -603,8 +605,8 @@ mod test { } #[test] - fn test_simplify_filter_multiple_column_in_or() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_multiple_column_in_or() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); // c1 > c2 or c1 > 1 let plan_1 = @@ -619,8 +621,9 @@ mod test { } #[test] - fn test_simplify_filter_multiple_dispersed_same_column_in_or() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_multiple_dispersed_same_column_in_or() -> Result<(), DatabaseError> + { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan_1 = table_state.plan_with_arena( "select * from t1 where c1 = 4 and c1 > c2 or c1 > 1", @@ -636,8 +639,8 @@ mod test { } #[test] - fn test_simplify_filter_column_is_null() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_column_is_null() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan_1 = table_state.plan_with_arena("select * from t1 where c1 is null", &mut arena)?; @@ -651,8 +654,8 @@ mod test { } #[test] - fn test_simplify_filter_column_is_not_null() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_column_is_not_null() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan_1 = table_state.plan_with_arena("select * from t1 where c1 is not null", &mut arena)?; @@ -669,8 +672,8 @@ mod test { } #[test] - fn constant_in_expansion_uses_original_list_length() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn constant_in_expansion_uses_original_list_length() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; for (list, kept) in [ ("1,2,3,4,5", false), ("1,2,3,4,5,6", true), @@ -699,8 +702,8 @@ mod test { } #[test] - fn unsafe_large_in_keeps_pairwise_expansion() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn unsafe_large_in_keeps_pairwise_expansion() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; for predicate in [ "c1 in (1,2,3,4,5,2147483648)", "c1 in (1,2,3,4,5,c1)", @@ -727,8 +730,8 @@ mod test { } #[test] - fn test_simplify_filter_column_in() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_column_in() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan_1 = table_state.plan_with_arena("select * from t1 where c1 in (1, 2, 3)", &mut arena)?; @@ -746,8 +749,8 @@ mod test { } #[test] - fn test_simplify_filter_column_not_in() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_simplify_filter_column_not_in() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let mut arena = PlanArena::new(&table_state.table_arena); let plan_1 = table_state .plan_with_arena("select * from t1 where c1 not in (1, 2, 3)", &mut arena)?; diff --git a/src/orm/ddl.rs b/src/orm/ddl.rs index 052d0493..1c363153 100644 --- a/src/orm/ddl.rs +++ b/src/orm/ddl.rs @@ -6,6 +6,7 @@ enum OrmDefaultChange { Drop, } +#[maybe_async] impl Database { fn table_catalog(&self, table_name: &str) -> Result, DatabaseError> { let transaction = self.storage.transaction()?; @@ -37,11 +38,11 @@ impl Database { /// let database = DataBaseBuilder::path(".").build_in_memory().unwrap(); /// database.create_table::().unwrap(); /// ``` - pub fn create_table(&mut self) -> Result<(), DatabaseError> { - execute_create_table::<_, M>(self, false)?; + pub async fn create_table(&mut self) -> Result<(), DatabaseError> { + execute_create_table::<_, M>(self, false).await?; for index in M::indexes() { - execute_create_index(self, M::table_name(), index, false)?; + execute_create_index(self, M::table_name(), index, false).await?; } Ok(()) @@ -52,11 +53,11 @@ impl Database { /// This is useful for examples, tests and bootstrap flows where rerunning /// schema initialization should stay idempotent. Secondary indexes declared /// with `#[model(index)]` are created with `IF NOT EXISTS` as well. - pub fn create_table_if_not_exists(&mut self) -> Result<(), DatabaseError> { - execute_create_table::<_, M>(self, true)?; + pub async fn create_table_if_not_exists(&mut self) -> Result<(), DatabaseError> { + execute_create_table::<_, M>(self, true).await?; for index in M::indexes() { - execute_create_index(self, M::table_name(), index, true)?; + execute_create_index(self, M::table_name(), index, true).await?; } Ok(()) @@ -82,12 +83,12 @@ impl Database { /// database.truncate::().unwrap(); /// assert_eq!(database.fetch::().unwrap().count(), 0); /// ``` - pub fn truncate(&self) -> Result<(), DatabaseError> { - self.bind(|ctx| ctx.truncate::())?.done() + pub async fn truncate(&self) -> Result<(), DatabaseError> { + self.bind(|ctx| ctx.truncate::()).await?.done().await } /// Creates a view from a binder-backed ORM plan builder. - pub fn create_view(&mut self, view_name: &str, build: F) -> Result<(), DatabaseError> + pub async fn create_view(&mut self, view_name: &str, build: F) -> Result<(), DatabaseError> where F: for<'ctx, 'bind, 'parent, 'arena> FnOnce( &'ctx mut OrmContext< @@ -100,11 +101,11 @@ impl Database { >, ) -> Result, { - execute_create_view(self, view_name, build, false) + execute_create_view(self, view_name, build, false).await } /// Creates or replaces a view from a binder-backed ORM plan builder. - pub fn create_or_replace_view( + pub async fn create_or_replace_view( &mut self, view_name: &str, build: F, @@ -121,17 +122,17 @@ impl Database { >, ) -> Result, { - execute_create_view(self, view_name, build, true) + execute_create_view(self, view_name, build, true).await } /// Drops a view by name. - pub fn drop_view(&mut self, view_name: &str) -> Result<(), DatabaseError> { - execute_drop_view(self, view_name, false) + pub async fn drop_view(&mut self, view_name: &str) -> Result<(), DatabaseError> { + execute_drop_view(self, view_name, false).await } /// Drops a view by name if it exists. - pub fn drop_view_if_exists(&mut self, view_name: &str) -> Result<(), DatabaseError> { - execute_drop_view(self, view_name, true) + pub async fn drop_view_if_exists(&mut self, view_name: &str) -> Result<(), DatabaseError> { + execute_drop_view(self, view_name, true).await } /// Migrates an existing table to match the current model definition. @@ -145,7 +146,7 @@ impl Database { /// to type, nullability and default expressions for non-primary-key columns /// when the underlying DDL supports them. Primary-key changes and unique /// constraint changes still return an error so you can handle them manually. - pub fn migrate(&mut self) -> Result<(), DatabaseError> { + pub async fn migrate(&mut self) -> Result<(), DatabaseError> { let fields = M::fields(); if fields.is_empty() { return Err(DatabaseError::UnsupportedStmt( @@ -155,7 +156,7 @@ impl Database { } let Some(table) = self.table_catalog(M::table_name())? else { - return self.create_table::(); + return self.create_table::().await; }; let (table_primary_key, current_columns) = { let table_arena = self.state.table_arena().borrow(); @@ -234,7 +235,8 @@ impl Database { field.data_type.clone(), OrmDefaultChange::NoChange, NotNullChange::NoChange, - )?; + ) + .await?; } let arena = PlanArena::new(self.state.table_arena()); @@ -250,7 +252,8 @@ impl Database { None => OrmDefaultChange::Drop, }, NotNullChange::NoChange, - )?; + ) + .await?; } if field.nullable != current_column.nullable() { @@ -266,7 +269,8 @@ impl Database { } else { NotNullChange::Set }, - )?; + ) + .await?; } } @@ -335,7 +339,8 @@ impl Database { current_column.datatype().clone(), OrmDefaultChange::NoChange, NotNullChange::NoChange, - )?; + ) + .await?; } for column in current_columns.values() { @@ -352,7 +357,7 @@ impl Database { ))); } - execute_drop_column(self, M::table_name(), column.name())?; + execute_drop_column(self, M::table_name(), column.name()).await?; } for field in fields { @@ -369,11 +374,11 @@ impl Database { ))); } - execute_add_column(self, M::table_name(), field)?; + execute_add_column(self, M::table_name(), field).await?; } for index in M::indexes() { - execute_create_index(self, M::table_name(), index, true)?; + execute_create_index(self, M::table_name(), index, true).await?; } Ok(()) @@ -383,16 +388,16 @@ impl Database { /// /// Primary-key indexes are managed by the table definition itself and /// cannot be dropped independently. - pub fn drop_index(&mut self, index_name: &str) -> Result<(), DatabaseError> { - execute_drop_index(self, M::table_name(), index_name, false) + pub async fn drop_index(&mut self, index_name: &str) -> Result<(), DatabaseError> { + execute_drop_index(self, M::table_name(), index_name, false).await } /// Drops a non-primary-key model index by name if it exists. - pub fn drop_index_if_exists( + pub async fn drop_index_if_exists( &mut self, index_name: &str, ) -> Result<(), DatabaseError> { - execute_drop_index(self, M::table_name(), index_name, true) + execute_drop_index(self, M::table_name(), index_name, true).await } /// Drops the model table. @@ -415,33 +420,37 @@ impl Database { /// database.create_table::().unwrap(); /// database.drop_table::().unwrap(); /// ``` - pub fn drop_table(&mut self) -> Result<(), DatabaseError> { - execute_drop_table(self, M::table_name(), false) + pub async fn drop_table(&mut self) -> Result<(), DatabaseError> { + execute_drop_table(self, M::table_name(), false).await } /// Drops the model table if it exists. /// /// This variant is convenient for cleanup code that should succeed even if /// the table was already removed. - pub fn drop_table_if_exists(&mut self) -> Result<(), DatabaseError> { - execute_drop_table(self, M::table_name(), true) + pub async fn drop_table_if_exists(&mut self) -> Result<(), DatabaseError> { + execute_drop_table(self, M::table_name(), true).await } } -fn execute_create_table( +#[maybe_async] +async fn execute_create_table( database: &mut Database, if_not_exists: bool, ) -> Result<(), DatabaseError> { - database.execute_mut("ORM CREATE TABLE", &[], move |binder, arena| { - let columns = M::fields() - .iter() - .map(|field| field.to_column_catalog(arena)) - .collect::, _>>()?; - binder.bind_create_table(M::table_name().into(), columns, if_not_exists) - }) + database + .execute_mut("ORM CREATE TABLE", &[], move |binder, arena| { + let columns = M::fields() + .iter() + .map(|field| field.to_column_catalog(arena)) + .collect::, _>>()?; + binder.bind_create_table(M::table_name().into(), columns, if_not_exists) + }) + .await } -fn execute_create_index( +#[maybe_async] +async fn execute_create_index( database: &mut Database, table_name: &'static str, index: &(&'static str, &'static [&'static str], bool), @@ -450,30 +459,33 @@ fn execute_create_index( let (index_name, index_columns, unique) = *index; let index_name = index_name.to_string(); let column_names = index_columns.to_vec(); - database.execute_mut("ORM CREATE INDEX", &[], move |binder, arena| { - let mut input = binder.bind_create_index_source(table_name, arena)?; - let schema = input.output_schema(arena).clone(); - let mut columns = Vec::with_capacity(column_names.len()); - for column_name in column_names { - let column = schema - .iter() - .copied() - .find(|column| arena.column(*column).name() == column_name) - .ok_or_else(|| DatabaseError::column_not_found(column_name.to_string()))?; - columns.push(column); - } - binder.bind_create_index( - table_name, - index_name, - columns, - if_not_exists, - unique, - input, - ) - }) + database + .execute_mut("ORM CREATE INDEX", &[], move |binder, arena| { + let mut input = binder.bind_create_index_source(table_name, arena)?; + let schema = input.output_schema(arena).clone(); + let mut columns = Vec::with_capacity(column_names.len()); + for column_name in column_names { + let column = schema + .iter() + .copied() + .find(|column| arena.column(*column).name() == column_name) + .ok_or_else(|| DatabaseError::column_not_found(column_name.to_string()))?; + columns.push(column); + } + binder.bind_create_index( + table_name, + index_name, + columns, + if_not_exists, + unique, + input, + ) + }) + .await } -fn execute_create_view( +#[maybe_async] +async fn execute_create_view( database: &mut Database, view_name: &str, build: F, @@ -493,32 +505,38 @@ where { static EMPTY_ORM_PARAMS: &[(usize, LogicalType)] = &[]; let view_name = view_name.to_string(); - database.execute_mut("ORM CREATE VIEW", EMPTY_ORM_PARAMS, move |binder, arena| { - let mut context = OrmContext { binder, arena }; - let plan = build(&mut context)?; - binder.bind_create_view( - view_name.as_str().into(), - or_replace, - plan, - vec![], - vec![], - arena, - ) - }) + database + .execute_mut("ORM CREATE VIEW", EMPTY_ORM_PARAMS, move |binder, arena| { + let mut context = OrmContext { binder, arena }; + let plan = build(&mut context)?; + binder.bind_create_view( + view_name.as_str().into(), + or_replace, + plan, + vec![], + vec![], + arena, + ) + }) + .await } -fn execute_drop_view( +#[maybe_async] +async fn execute_drop_view( database: &mut Database, view_name: &str, if_exists: bool, ) -> Result<(), DatabaseError> { let view_name = view_name.to_string(); - database.execute_mut("ORM DROP VIEW", &[], move |binder, _| { - binder.bind_drop_view(view_name.as_str().into(), if_exists) - }) + database + .execute_mut("ORM DROP VIEW", &[], move |binder, _| { + binder.bind_drop_view(view_name.as_str().into(), if_exists) + }) + .await } -fn execute_change_column( +#[maybe_async] +async fn execute_change_column( database: &mut Database, table_name: &'static str, old_column_name: &str, @@ -529,63 +547,77 @@ fn execute_change_column( ) -> Result<(), DatabaseError> { let old_column_name = old_column_name.to_string(); let new_column_name = new_column_name.to_string(); - database.execute_mut("ORM CHANGE COLUMN", &[], move |binder, arena| { - let default_change = match default_change { - OrmDefaultChange::Set(expr) => DefaultChange::Set(arena.alloc_expression(expr)), - OrmDefaultChange::Drop => DefaultChange::Drop, - OrmDefaultChange::NoChange => DefaultChange::NoChange, - }; - binder.bind_change_column( - table_name, - old_column_name, - new_column_name, - data_type, - default_change, - not_null_change, - ) - }) + database + .execute_mut("ORM CHANGE COLUMN", &[], move |binder, arena| { + let default_change = match default_change { + OrmDefaultChange::Set(expr) => DefaultChange::Set(arena.alloc_expression(expr)), + OrmDefaultChange::Drop => DefaultChange::Drop, + OrmDefaultChange::NoChange => DefaultChange::NoChange, + }; + binder.bind_change_column( + table_name, + old_column_name, + new_column_name, + data_type, + default_change, + not_null_change, + ) + }) + .await } -fn execute_drop_column( +#[maybe_async] +async fn execute_drop_column( database: &mut Database, table_name: &'static str, column_name: &str, ) -> Result<(), DatabaseError> { let column_name = column_name.to_string(); - database.execute_mut("ORM DROP COLUMN", &[], move |binder, _| { - binder.bind_drop_column(table_name, column_name, false) - }) + database + .execute_mut("ORM DROP COLUMN", &[], move |binder, _| { + binder.bind_drop_column(table_name, column_name, false) + }) + .await } -fn execute_add_column( +#[maybe_async] +async fn execute_add_column( database: &mut Database, table_name: &'static str, field: &OrmField, ) -> Result<(), DatabaseError> { let field = field.clone(); - database.execute_mut("ORM ADD COLUMN", &[], move |binder, arena| { - binder.bind_add_column(table_name, field.to_column_catalog(arena)?, false) - }) + database + .execute_mut("ORM ADD COLUMN", &[], move |binder, arena| { + binder.bind_add_column(table_name, field.to_column_catalog(arena)?, false) + }) + .await } -fn execute_drop_index( +#[maybe_async] +async fn execute_drop_index( database: &mut Database, table_name: &'static str, index_name: &str, if_exists: bool, ) -> Result<(), DatabaseError> { let index_name = index_name.to_string(); - database.execute_mut("ORM DROP INDEX", &[], move |binder, _| { - binder.bind_drop_index(table_name.into(), index_name, if_exists) - }) + database + .execute_mut("ORM DROP INDEX", &[], move |binder, _| { + binder.bind_drop_index(table_name.into(), index_name, if_exists) + }) + .await } -fn execute_drop_table( +#[maybe_async] +async fn execute_drop_table( database: &mut Database, table_name: &'static str, if_exists: bool, ) -> Result<(), DatabaseError> { - database.execute_mut("ORM DROP TABLE", &[], move |binder, _| { - binder.bind_drop_table(table_name.into(), if_exists) - }) + database + .execute_mut("ORM DROP TABLE", &[], move |binder, _| { + binder.bind_drop_table(table_name.into(), if_exists) + }) + .await } diff --git a/src/orm/dml.rs b/src/orm/dml.rs index 1cb39aa3..bda6753e 100644 --- a/src/orm/dml.rs +++ b/src/orm/dml.rs @@ -1,13 +1,14 @@ use super::*; use std::borrow::Borrow; +#[maybe_async] impl Database { /// Refreshes optimizer statistics for the model table. /// /// This runs `ANALYZE TABLE` for the backing table so the optimizer can use /// up-to-date statistics. - pub fn analyze_model(&mut self) -> Result<(), DatabaseError> { - self.analyze(M::table_name()) + pub async fn analyze_model(&mut self) -> Result<(), DatabaseError> { + self.analyze(M::table_name()).await } /// Inserts a model into its backing table. @@ -30,8 +31,8 @@ impl Database { /// database.create_table::().unwrap(); /// database.insert(&User { id: 1, name: "Alice".to_string() }).unwrap(); /// ``` - pub fn insert(&self, model: &M) -> Result<(), DatabaseError> { - orm_insert::<_, M>(self, model) + pub async fn insert(&self, model: &M) -> Result<(), DatabaseError> { + orm_insert::<_, M>(self, model).await } /// Inserts multiple models into their backing table. @@ -58,39 +59,40 @@ impl Database { /// .unwrap(); /// assert_eq!(database.fetch::().unwrap().count(), 2); /// ``` - pub fn insert_many(&self, models: I) -> Result<(), DatabaseError> + pub async fn insert_many(&self, models: I) -> Result<(), DatabaseError> where M: Model, I: IntoIterator, B: Borrow, { for model in models { - orm_insert::<_, M>(self, model.borrow())?; + orm_insert::<_, M>(self, model.borrow()).await?; } Ok(()) } } +#[maybe_async] impl<'a, S: Storage> Transaction<'a, S> { /// Refreshes optimizer statistics for the model table inside the current transaction. - pub fn analyze(&mut self) -> Result<(), DatabaseError> { - orm_analyze::<_, M>(self) + pub async fn analyze(&mut self) -> Result<(), DatabaseError> { + orm_analyze::<_, M>(self).await } /// Inserts a model inside the current transaction. - pub fn insert(&mut self, model: &M) -> Result<(), DatabaseError> { - orm_insert::<_, M>(self, model) + pub async fn insert(&mut self, model: &M) -> Result<(), DatabaseError> { + orm_insert::<_, M>(self, model).await } /// Inserts multiple models inside the current transaction. - pub fn insert_many(&mut self, models: I) -> Result<(), DatabaseError> + pub async fn insert_many(&mut self, models: I) -> Result<(), DatabaseError> where M: Model, I: IntoIterator, B: Borrow, { for model in models { - orm_insert::<_, M>(&mut *self, model.borrow())?; + orm_insert::<_, M>(&mut *self, model.borrow()).await?; } Ok(()) } diff --git a/src/orm/dql.rs b/src/orm/dql.rs index b63c8ba8..7f6e7afd 100644 --- a/src/orm/dql.rs +++ b/src/orm/dql.rs @@ -1,5 +1,6 @@ use super::*; +#[maybe_async] impl Database { /// Prepares an ORM-built plan once into a program for repeated execution with /// positional parameters. Use `e.param(id)` inside expression closures, then pass @@ -32,20 +33,22 @@ impl Database { } /// Executes a binder-backed plan built inside a closure. - pub fn bind(&self, build: F) -> Result, DatabaseError> + pub async fn bind(&self, build: F) -> Result, DatabaseError> where F: for<'ctx, 'bind, 'parent, 'arena> FnOnce( - &'ctx mut OrmContext< - 'ctx, - 'bind, - 'parent, - 'arena, - S::TransactionType<'_>, - &'static [(usize, LogicalType)], - >, - ) -> Result, + &'ctx mut OrmContext< + 'ctx, + 'bind, + 'parent, + 'arena, + S::TransactionType<'_>, + &'static [(usize, LogicalType)], + >, + ) + -> Result + + MaybeSend, { - bind_orm_context(self, false, build) + bind_orm_context(self, false, build).await } /// Explains a binder-backed plan built inside a closure. @@ -90,8 +93,8 @@ impl Database { /// let user = database.get::(&1).unwrap().unwrap(); /// assert_eq!(user.name, "Alice"); /// ``` - pub fn get(&self, key: &M::PrimaryKey) -> Result, DatabaseError> { - orm_get::<_, M>(self, key) + pub async fn get(&self, key: &M::PrimaryKey) -> Result, DatabaseError> { + orm_get::<_, M>(self, key).await } /// Fetches all rows from the model table as a typed iterator. @@ -118,8 +121,12 @@ impl Database { /// assert_eq!(users.len(), 1); /// assert_eq!(users[0].name, "Alice"); /// ``` - pub fn fetch(&self) -> Result, M>, DatabaseError> { - Ok(bind_orm_context(self, true, |ctx| ctx.from::()?.finish())?.orm::()) + pub async fn fetch(&self) -> Result, M>, DatabaseError> { + Ok( + bind_orm_context(self, true, |ctx| ctx.from::()?.finish()) + .await? + .orm::(), + ) } /// Lists all table names. @@ -141,20 +148,20 @@ impl Database { /// let tables = database.show_tables().unwrap().collect::, _>>().unwrap(); /// assert!(tables.iter().any(|name| name == "users")); /// ``` - pub fn show_tables( + pub async fn show_tables( &self, ) -> Result, String>, DatabaseError> { Ok(ProjectValueIter::new( - self.bind(|ctx| ctx.binder.bind_show_tables())?, + self.bind(|ctx| ctx.binder.bind_show_tables()).await?, )) } /// Lists all view names. - pub fn show_views( + pub async fn show_views( &self, ) -> Result, String>, DatabaseError> { Ok(ProjectValueIter::new( - self.bind(|ctx| ctx.binder.bind_show_views())?, + self.bind(|ctx| ctx.binder.bind_show_views()).await?, )) } @@ -177,15 +184,17 @@ impl Database { /// let columns = database.describe::().unwrap().collect::, _>>().unwrap(); /// assert!(columns.iter().any(|column| column.field == "id")); /// ``` - pub fn describe( + pub async fn describe( &self, ) -> Result, DescribeColumn>, DatabaseError> { Ok(self - .bind(|ctx| ctx.binder.bind_describe(M::table_name()))? + .bind(|ctx| ctx.binder.bind_describe(M::table_name())) + .await? .orm::()) } } +#[maybe_async] impl<'a, S: Storage> Transaction<'a, S> { /// Prepares an ORM-built plan into a program, against this transaction's view. pub fn program( @@ -213,23 +222,25 @@ impl<'a, S: Storage> Transaction<'a, S> { } /// Executes a binder-backed plan inside the current transaction. - pub fn bind( + pub async fn bind( &mut self, build: F, ) -> Result>, DatabaseError> where F: for<'ctx, 'bind, 'parent, 'arena> FnOnce( - &'ctx mut OrmContext< - 'ctx, - 'bind, - 'parent, - 'arena, - S::TransactionType<'a>, - &'static [(usize, LogicalType)], - >, - ) -> Result, + &'ctx mut OrmContext< + 'ctx, + 'bind, + 'parent, + 'arena, + S::TransactionType<'a>, + &'static [(usize, LogicalType)], + >, + ) + -> Result + + MaybeSend, { - bind_orm_context(self, false, build) + bind_orm_context(self, false, build).await } /// Explains a binder-backed plan inside the current transaction. @@ -251,13 +262,16 @@ impl<'a, S: Storage> Transaction<'a, S> { /// Loads a single model by primary key inside the current transaction. A model with a /// lifetime may borrow its strings and binaries from the transaction. - pub fn get<'t, M: Model + FromTxnRow<'t>>( + pub async fn get<'t, M: Model + FromTxnRow<'t>>( &'t mut self, key: &M::PrimaryKey, ) -> Result, DatabaseError> { - let mut iter = bind_get::<_, M>(self, key)?; + let mut iter = bind_get::<_, M>(self, key).await?; Ok( - match iter.next_tuple(|schema, row| M::from_txn_row(schema, row))? { + match iter + .next_tuple(|schema, row| M::from_txn_row(schema, row)) + .await? + { Some(row) => Some(row?), None => None, }, @@ -266,46 +280,51 @@ impl<'a, S: Storage> Transaction<'a, S> { /// Fetches all rows for a model inside the current transaction. A model with a lifetime may /// borrow its strings and binaries from the transaction. - pub fn fetch<'t, M: Model + FromTxnRow<'t>>( + pub async fn fetch<'t, M: Model + FromTxnRow<'t>>( &'t mut self, ) -> Result>, M>, DatabaseError> { - Ok(bind_orm_context(self, true, |ctx| ctx.from::()?.finish())?.txn_orm::()) + Ok( + bind_orm_context(self, true, |ctx| ctx.from::()?.finish()) + .await? + .txn_orm::(), + ) } /// Lists all table names inside the current transaction. - pub fn show_tables( + pub async fn show_tables( &mut self, ) -> Result< ProjectValueIter>, String>, DatabaseError, > { Ok(ProjectValueIter::new( - self.bind(|ctx| ctx.binder.bind_show_tables())?, + self.bind(|ctx| ctx.binder.bind_show_tables()).await?, )) } /// Lists all view names inside the current transaction. - pub fn show_views( + pub async fn show_views( &mut self, ) -> Result< ProjectValueIter>, String>, DatabaseError, > { Ok(ProjectValueIter::new( - self.bind(|ctx| ctx.binder.bind_show_views())?, + self.bind(|ctx| ctx.binder.bind_show_views()).await?, )) } /// Describes the schema of the model table inside the current transaction. - pub fn describe( + pub async fn describe( &mut self, ) -> Result< OrmIter>, DescribeColumn>, DatabaseError, > { Ok(self - .bind(|ctx| ctx.binder.bind_describe(M::table_name()))? + .bind(|ctx| ctx.binder.bind_describe(M::table_name())) + .await? .orm::()) } } diff --git a/src/orm/mod.rs b/src/orm/mod.rs index 595cac1f..6227cd4d 100644 --- a/src/orm/mod.rs +++ b/src/orm/mod.rs @@ -6,13 +6,14 @@ use crate::binder::{ }; use crate::catalog::{ColumnCatalog, ColumnRef, TableCatalog}; use crate::db::{ - BindSource, Database, DatabaseIter, Executor, OrmIter, ResultIter, ResultRow, Transaction, - TransactionIter, TxnOrmIter, TxnResultIter, TxnValue, + AsyncIter, BindSource, Database, DatabaseIter, Executor, OrmIter, ResultIter, ResultRow, + Transaction, TransactionIter, TxnOrmIter, TxnResultIter, TxnValue, }; use crate::errors::DatabaseError; pub use crate::expression::agg::AggKind; use crate::expression::window::WindowFunctionKind; use crate::expression::{self, AliasType, ScalarExpression, TypeCast}; +use crate::maybe_async::MaybeSend; use crate::planner::operator::alter_table::change_column::{DefaultChange, NotNullChange}; use crate::planner::operator::join::JoinType; use crate::planner::operator::mark_apply::MarkApplyQuantifier; @@ -1030,38 +1031,42 @@ where } } -fn bind_orm_context<'a, E, F>( +#[maybe_async] +async fn bind_orm_context<'a, E, F>( executor: E, read_only: bool, build: F, ) -> Result where - E: BindSource<'a>, + E: BindSource, F: for<'ctx, 'bind, 'parent, 'arena> FnOnce( - &'ctx mut OrmContext< - 'ctx, - 'bind, - 'parent, - 'arena, - E::Transaction, - &'static [(usize, LogicalType)], - >, - ) -> Result, + &'ctx mut OrmContext< + 'ctx, + 'bind, + 'parent, + 'arena, + E::Transaction, + &'static [(usize, LogicalType)], + >, + ) -> Result + + MaybeSend, { static EMPTY_BIND_PARAMS: &[(usize, LogicalType)] = &[]; - executor.execute(read_only, |state, tx| { - state - .build_plan(EMPTY_BIND_PARAMS, tx, |binder, arena| { - let mut context = OrmContext { binder, arena }; - build(&mut context) - }) - .map(Executor::from_plan) - }) + executor + .execute(read_only, |state, tx| { + state + .build_plan(EMPTY_BIND_PARAMS, tx, |binder, arena| { + let mut context = OrmContext { binder, arena }; + build(&mut context) + }) + .map(Executor::from_plan) + }) + .await } fn explain_orm_context<'a, E, F>(executor: E, build: F) -> Result where - E: BindSource<'a>, + E: BindSource, F: for<'ctx, 'bind, 'parent, 'arena> FnOnce( &'ctx mut OrmContext< 'ctx, @@ -2771,7 +2776,7 @@ pub trait Model: Sized + FromQueryRow { /// .project_value::(); /// # Ok::<(), kite_sql::errors::DatabaseError>(()) /// ``` -pub trait FromDataValue: Sized { +pub trait FromDataValue: Sized + MaybeSend { /// Returns the logical SQL type used for conversion, when one is required. fn logical_type() -> Option; @@ -2789,7 +2794,7 @@ pub trait FromDataValue: Sized { /// .project_tuple::<(i32, String)>(); /// # Ok::<(), kite_sql::errors::DatabaseError>(()) /// ``` -pub trait FromQueryTuple: Sized { +pub trait FromQueryTuple: Sized + MaybeSend { /// Decodes one projected tuple into `Self`. fn from_query_tuple(tuple: &mut ResultRow<'_, '_>) -> Result; } @@ -2797,7 +2802,7 @@ pub trait FromQueryTuple: Sized { /// Conversion trait from a query result row into a Rust value. /// /// `#[derive(Model)]` and `#[derive(Projection)]` generate this automatically. -pub trait FromQueryRow: Sized { +pub trait FromQueryRow: Sized + MaybeSend { /// Decodes one result row into `Self`. fn from_query_row( schema: &SchemaView<'_, '_>, @@ -2805,7 +2810,7 @@ pub trait FromQueryRow: Sized { ) -> Result; } -pub trait FromTxnRow<'txn>: Sized { +pub trait FromTxnRow<'txn>: Sized + MaybeSend { fn from_txn_row( schema: &SchemaView<'_, '_>, row: &mut ResultRow<'_, 'txn>, @@ -2892,6 +2897,7 @@ pub trait OrmQueryResultExt: ResultIter + Sized { impl OrmQueryResultExt for I {} +#[maybe_async] impl ProjectValueIter where I: ResultIter, @@ -2908,8 +2914,28 @@ where /// /// This is useful when you stop iterating early and want to release the /// underlying result stream. - pub fn done(self) -> Result<(), DatabaseError> { - self.inner.done() + pub async fn done(self) -> Result<(), DatabaseError> { + self.inner.done().await + } +} + +#[maybe_async] +impl AsyncIter for ProjectValueIter +where + I: ResultIter + MaybeSend, + T: FromDataValue + MaybeSend, +{ + #[async_only] + type Item = Result; + + /// Reads the next row. + #[async_only] + async fn next(&mut self) -> Option { + self.inner + .next_tuple(|_, tuple| extract_value_from_tuple::(tuple)) + .await + .transpose() + .map(|value| value.and_then(std::convert::identity)) } } @@ -2932,6 +2958,7 @@ pub struct ProjectTupleIter { _marker: PhantomData, } +#[maybe_async] impl ProjectTupleIter where I: ResultIter, @@ -2948,11 +2975,33 @@ where /// /// This is useful when you stop iterating early and want to release the /// underlying result stream. - pub fn done(self) -> Result<(), DatabaseError> { - self.inner.done() + pub async fn done(self) -> Result<(), DatabaseError> { + self.inner.done().await + } +} + +#[maybe_async] +impl AsyncIter for ProjectTupleIter +where + I: ResultIter + MaybeSend, + T: FromQueryTuple + MaybeSend, +{ + #[async_only] + type Item = Result; + + /// Reads the next row. + #[async_only] + async fn next(&mut self) -> Option { + self.inner + .next_tuple(|_, tuple| extract_projected_tuple::(tuple)) + .await + .transpose() + .map(|tuple| tuple.and_then(std::convert::identity)) } } +#[maybe_async] +#[sync_only] impl Iterator for ProjectValueIter where I: ResultIter, @@ -2969,6 +3018,8 @@ where } } +#[maybe_async] +#[sync_only] impl Iterator for ProjectTupleIter where I: ResultIter, @@ -3397,21 +3448,26 @@ fn model_column_rename_compatible( && model_column_default(model) == catalog_column_default(column, arena)) } -fn extract_optional_model(iter: I) -> Result, DatabaseError> +#[maybe_async] +async fn extract_optional_model(iter: I) -> Result, DatabaseError> where I: ResultIter, M: Model, { - extract_optional_row(iter) + extract_optional_row(iter).await } -fn extract_optional_row(mut iter: I) -> Result, DatabaseError> +#[maybe_async] +async fn extract_optional_row(mut iter: I) -> Result, DatabaseError> where I: ResultIter, T: FromQueryRow, { Ok( - match iter.next_tuple(|schema, tuple| T::from_query_row(schema, tuple))? { + match iter + .next_tuple(|schema, tuple| T::from_query_row(schema, tuple)) + .await? + { Some(row) => Some(row?), None => None, }, @@ -3471,7 +3527,8 @@ fn extract_projected_tuple( T::from_query_tuple(tuple) } -fn orm_analyze<'a, E: BindSource<'a>, M: Model>(executor: E) -> Result<(), DatabaseError> { +#[maybe_async] +async fn orm_analyze(executor: E) -> Result<(), DatabaseError> { executor .execute(false, |state, tx| { state @@ -3479,14 +3536,14 @@ fn orm_analyze<'a, E: BindSource<'a>, M: Model>(executor: E) -> Result<(), Datab binder.bind_analyze(M::table_name(), arena) }) .map(Executor::from_plan) - })? + }) + .await? .done() + .await } -fn orm_insert<'a, E: BindSource<'a>, M: Model>( - executor: E, - model: &M, -) -> Result<(), DatabaseError> { +#[maybe_async] +async fn orm_insert(executor: E, model: &M) -> Result<(), DatabaseError> { let params = model.params(); executor .execute(false, |state, tx| { @@ -3495,19 +3552,23 @@ fn orm_insert<'a, E: BindSource<'a>, M: Model>( bind_orm_insert_model::<_, _, M>(binder, params, arena) }) .map(Executor::from_plan) - })? + }) + .await? .done() + .await } -fn orm_get<'a, E: BindSource<'a>, M: Model>( +#[maybe_async] +async fn orm_get( executor: E, key: &M::PrimaryKey, ) -> Result, DatabaseError> { - extract_optional_model(bind_get::<_, M>(executor, key)?) + extract_optional_model(bind_get::<_, M>(executor, key).await?).await } /// Binds the lookup of one `M` by primary key. -fn bind_get<'a, E: BindSource<'a>, M: Model>( +#[maybe_async] +async fn bind_get( executor: E, key: &M::PrimaryKey, ) -> Result { @@ -3526,12 +3587,36 @@ fn bind_get<'a, E: BindSource<'a>, M: Model>( .finish()?; Ok(plan) }) + .await } // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; + + /// Reads every row of an ORM iterator, on either path. + #[keep_async_fn] + trait TryCollect { + async fn try_collect(self) -> Result, DatabaseError>; + } + + impl TryCollect for I + where + I: AsyncIter>, + { + // `next` is `Iterator::next` on the sync path only. + #[allow(clippy::while_let_on_iterator)] + async fn try_collect(mut self) -> Result, DatabaseError> { + let mut rows = Vec::new(); + while let Some(row) = self.next().await { + rows.push(row?); + } + Ok(rows) + } + } + use crate::expression::BinaryOperator; use crate::types::tuple::{Schema, SchemaView, Tuple, TupleImpl}; use std::sync::Arc; @@ -3724,13 +3809,17 @@ mod tests { } } - fn build_orm_unit_database( - ) -> Result, DatabaseError> { - let mut database = crate::db::DataBaseBuilder::path("./orm-unit-test").build_in_memory()?; + async fn build_orm_unit_database( + ) -> Result, DatabaseError> { + let mut database = crate::db::DataBaseBuilder::path("./orm-unit-test") + .build_in_memory() + .await?; database - .ddl("create table orm_unit_users (id int primary key, name varchar, age int null)")?; + .ddl("create table orm_unit_users (id int primary key, name varchar, age int null)") + .await?; database - .ddl("create table orm_unit_orders (id int primary key, user_id int, amount int)")?; + .ddl("create table orm_unit_orders (id int primary key, user_id int, amount int)") + .await?; for user in [ OrmUnitUser { @@ -3749,7 +3838,7 @@ mod tests { age: None, }, ] { - database.insert(&user)?; + database.insert(&user).await?; } for order in [ @@ -3769,7 +3858,7 @@ mod tests { amount: 200, }, ] { - database.insert(&order)?; + database.insert(&order).await?; } Ok(database) @@ -3941,15 +4030,20 @@ mod tests { } #[test] - fn prepared_orm_query_reuses_plan_and_binds_each_execution() -> Result<(), DatabaseError> { - let mut database = crate::db::DataBaseBuilder::path("./orm-unit-test").build_in_memory()?; - database.create_table::()?; + async fn prepared_orm_query_reuses_plan_and_binds_each_execution() -> Result<(), DatabaseError> + { + let mut database = crate::db::DataBaseBuilder::path("./orm-unit-test") + .build_in_memory() + .await?; + database.create_table::().await?; for (id, name) in [(1, "Alice"), (2, "Bob"), (3, "Cara")] { - database.insert(&OrmUnitUser { - id, - name: name.into(), - age: Some(18), - })?; + database + .insert(&OrmUnitUser { + id, + name: name.into(), + age: Some(18), + }) + .await?; } let types = [(1, LogicalType::Integer)]; assert!(matches!( @@ -3968,9 +4062,11 @@ mod tests { let mut tx = database.new_transaction(false)?; for (id, expected) in [(1, "Alice"), (3, "Cara"), (2, "Bob")] { let rows = plan - .execute(&mut tx, [(1, DataValue::Int32(id).into())])? + .execute(&mut tx, [(1, DataValue::Int32(id).into())]) + .await? .orm::() - .collect::, _>>()?; + .try_collect() + .await?; assert_eq!(rows.len(), 1); assert_eq!(rows[0].name, expected); } @@ -3980,51 +4076,62 @@ mod tests { .finish() })?; let rows = tx_plan - .execute(&mut tx, [(1, DataValue::Int32(2).into())])? + .execute(&mut tx, [(1, DataValue::Int32(2).into())]) + .await? .orm::() - .collect::, _>>()?; + .try_collect() + .await?; assert_eq!(rows[0].name, "Bob"); - tx.commit()?; + tx.commit().await?; Ok(()) } #[test] - fn database_and_transaction_orm_helpers_bind_expected_plans() -> Result<(), DatabaseError> { + async fn database_and_transaction_orm_helpers_bind_expected_plans() -> Result<(), DatabaseError> + { let mut database = crate::db::DataBaseBuilder::path("./orm-unit-test") .histogram_buckets(100) - .build_in_memory()?; + .build_in_memory() + .await?; database - .ddl("create table orm_unit_users (id int primary key, name varchar, age int null)")?; - database.create_view("orm_unit_user_names", |ctx| { - ctx.from::()? - .project_scalars((OrmUnitUser::id(), OrmUnitUser::name()))? - .finish() - })?; + .ddl("create table orm_unit_users (id int primary key, name varchar, age int null)") + .await?; + database + .create_view("orm_unit_user_names", |ctx| { + ctx.from::()? + .project_scalars((OrmUnitUser::id(), OrmUnitUser::name()))? + .finish() + }) + .await?; - database.insert(&OrmUnitUser { - id: 1, - name: "Alice".to_string(), - age: Some(18), - })?; - database.insert_many([ - OrmUnitUser { - id: 2, - name: "Bob".to_string(), - age: Some(20), - }, - OrmUnitUser { - id: 3, - name: "Cara".to_string(), - age: None, - }, - ])?; + database + .insert(&OrmUnitUser { + id: 1, + name: "Alice".to_string(), + age: Some(18), + }) + .await?; + database + .insert_many([ + OrmUnitUser { + id: 2, + name: "Bob".to_string(), + age: Some(20), + }, + OrmUnitUser { + id: 3, + name: "Cara".to_string(), + age: None, + }, + ]) + .await?; assert!(matches!( - database.analyze_model::(), + database.analyze_model::().await, Err(DatabaseError::TooManyBuckets(100, 3)) )); assert_eq!( - database.get::(&1)?, + database.get::(&1).await?, Some(OrmUnitUser { id: 1, name: "Alice".to_string(), @@ -4033,19 +4140,25 @@ mod tests { ); assert_eq!( database - .fetch::()? - .collect::, _>>()? + .fetch::() + .await? + .try_collect() + .await? .len(), 3 ); assert!(database - .show_views()? - .collect::, _>>()? + .show_views() + .await? + .try_collect() + .await? .iter() .any(|view| view == "orm_unit_user_names")); assert!(database - .describe::()? - .collect::, _>>()? + .describe::() + .await? + .try_collect() + .await? .iter() .any(|column| column.field == "name")); @@ -4054,7 +4167,8 @@ mod tests { id: 4, name: "Dora".to_string(), age: Some(40), - })?; + }) + .await?; tx.insert_many([ OrmUnitUser { id: 5, @@ -4066,32 +4180,37 @@ mod tests { name: "Finn".to_string(), age: Some(60), }, - ])?; + ]) + .await?; assert!(matches!( - tx.analyze::(), + tx.analyze::().await, Err(DatabaseError::TooManyBuckets(100, 6)) )); - assert_eq!(tx.get::(&4)?.unwrap().name, "Dora"); + assert_eq!(tx.get::(&4).await?.unwrap().name, "Dora"); assert_eq!( - tx.fetch::()? - .collect::, _>>()? - .len(), + tx.fetch::().await?.try_collect().await?.len(), 6 ); assert!(tx - .show_tables()? - .collect::, _>>()? + .show_tables() + .await? + .try_collect() + .await? .iter() .any(|table| table == "orm_unit_users")); assert!(tx - .show_views()? - .collect::, _>>()? + .show_views() + .await? + .try_collect() + .await? .iter() .any(|view| view == "orm_unit_user_names")); assert!(tx - .describe::()? - .collect::, _>>()? + .describe::() + .await? + .try_collect() + .await? .iter() .any(|column| column.field == "age")); @@ -4111,15 +4230,15 @@ mod tests { "{plan}" ); - tx.commit()?; + tx.commit().await?; Ok(()) } #[test] - fn expression_scope_helpers_build_filter_projection_and_sort_exprs() -> Result<(), DatabaseError> - { - let database = build_orm_unit_database()?; + async fn expression_scope_helpers_build_filter_projection_and_sort_exprs( + ) -> Result<(), DatabaseError> { + let database = build_orm_unit_database().await?; let expression_plan = database.explain(|ctx| { ctx.from::()? @@ -4197,9 +4316,9 @@ mod tests { } #[test] - fn chained_range_and_function_helpers_preserve_values_and_null_rows( + async fn chained_range_and_function_helpers_preserve_values_and_null_rows( ) -> Result<(), DatabaseError> { - let database = build_orm_unit_database()?; + let database = build_orm_unit_database().await?; let mut iter = database .bind(|ctx| { ctx.from::()? @@ -4213,12 +4332,13 @@ mod tests { })? .order_by_expr(|e| Ok(e.column(OrmUnitUser::id())?.asc()))? .finish() - })? + }) + .await? .project_tuple::<(i32, String)>(); - assert_eq!(iter.next().transpose()?, Some((1, "ALICE".into()))); - assert_eq!(iter.next().transpose()?, Some((2, "BOB".into()))); - assert!(iter.next().transpose()?.is_none()); - iter.done()?; + assert_eq!(iter.next().await.transpose()?, Some((1, "ALICE".into()))); + assert_eq!(iter.next().await.transpose()?, Some((2, "BOB".into()))); + assert!(iter.next().await.transpose()?.is_none()); + iter.done().await?; let mut iter = database .bind(|ctx| { ctx.from::()? @@ -4228,17 +4348,18 @@ mod tests { })? .project_scalar(OrmUnitUser::id())? .finish() - })? + }) + .await? .project_value::(); - assert_eq!(iter.next().transpose()?, Some(3)); - assert!(iter.next().transpose()?.is_none()); - iter.done()?; + assert_eq!(iter.next().await.transpose()?, Some(3)); + assert!(iter.next().await.transpose()?.is_none()); + iter.done().await?; Ok(()) } #[test] - fn insert_select_maps_columns_and_rejects_invalid_targets() -> Result<(), DatabaseError> { - let database = build_orm_unit_database()?; + async fn insert_select_maps_columns_and_rejects_invalid_targets() -> Result<(), DatabaseError> { + let database = build_orm_unit_database().await?; database .bind(|ctx| { ctx.insert_select::(["name", "id", "age"], |ctx| { @@ -4253,8 +4374,10 @@ mod tests { })? .finish() }) - })? - .done()?; + }) + .await? + .done() + .await?; database .bind(|ctx| { ctx.insert_select::(std::iter::empty::(), |ctx| { @@ -4269,16 +4392,20 @@ mod tests { })? .finish() }) - })? - .done()?; + }) + .await? + .done() + .await?; assert!(matches!( - database.bind( - |ctx| ctx.insert_select::(["id", "name"], |ctx| { - ctx.from::()? - .project_scalar(OrmUnitUser::id())? - .finish() - }) - ), + database + .bind( + |ctx| ctx.insert_select::(["id", "name"], |ctx| { + ctx.from::()? + .project_scalar(OrmUnitUser::id())? + .finish() + }) + ) + .await, Err(DatabaseError::ValuesLenMismatch(2, 1)) )); assert!(database @@ -4289,12 +4416,18 @@ mod tests { .finish() }) ) + .await .is_err()); - let mut iter = - database.run("select id,name,age from orm_unit_users where id>3 order by id")?; + let mut iter = database + .run("select id,name,age from orm_unit_users where id>3 order by id") + .await?; let mut rows = Vec::new(); - while iter.next_tuple(|_, row| rows.push(row.clone()))?.is_some() {} - iter.done()?; + while iter + .next_tuple(|_, row| rows.push(row.clone())) + .await? + .is_some() + {} + iter.done().await?; assert_eq!( rows, vec![ @@ -4319,8 +4452,9 @@ mod tests { } #[test] - fn case_helpers_infer_else_types_and_preserve_null_fallbacks() -> Result<(), DatabaseError> { - let database = build_orm_unit_database()?; + async fn case_helpers_infer_else_types_and_preserve_null_fallbacks() -> Result<(), DatabaseError> + { + let database = build_orm_unit_database().await?; let mut rows = database .bind(|ctx| { ctx.from::()? @@ -4338,23 +4472,24 @@ mod tests { })? .order_by(OrmUnitUser::id().asc())? .finish() - })? + }) + .await? .project_tuple::<(i32, i32, String, Option)>(); for expected in [ (1, 7, "fallback".into(), None), (2, 7, "fallback".into(), Some(1)), (3, 7, "fallback".into(), None), ] { - assert_eq!(rows.next().transpose()?, Some(expected)); + assert_eq!(rows.next().await.transpose()?, Some(expected)); } - assert!(rows.next().transpose()?.is_none()); - rows.done()?; + assert!(rows.next().await.transpose()?.is_none()); + rows.done().await?; Ok(()) } #[test] - fn grouped_orm_having_reads_the_aggregate_output() -> Result<(), DatabaseError> { - let database = build_orm_unit_database()?; + async fn grouped_orm_having_reads_the_aggregate_output() -> Result<(), DatabaseError> { + let database = build_orm_unit_database().await?; let mut rows = database .bind(|ctx| { ctx.from::()? @@ -4366,31 +4501,36 @@ mod tests { .having(|e| e.column(OrmUnitOrder::user_id())?.eq(1))? .order_by(OrmUnitOrder::user_id().asc())? .finish() - })? + }) + .await? .project_tuple::<(i32, i32)>(); - assert_eq!(rows.next().transpose()?, Some((1, 2))); - assert!(rows.next().transpose()?.is_none()); - rows.done()?; + assert_eq!(rows.next().await.transpose()?, Some((1, 2))); + assert!(rows.next().await.transpose()?.is_none()); + rows.done().await?; assert!(matches!( - database.bind(|ctx| { - ctx.from::()? - .project_tuple(|e| { - Ok(vec![e.column(OrmUnitOrder::user_id())?, e.count_all()?]) - })? - .group_by(|e| e.column(OrmUnitOrder::user_id()))? - .having(|e| e.column(OrmUnitOrder::amount())?.gt(100))? - .finish() - }), + database + .bind(|ctx| { + ctx.from::()? + .project_tuple(|e| { + Ok(vec![e.column(OrmUnitOrder::user_id())?, e.count_all()?]) + })? + .group_by(|e| e.column(OrmUnitOrder::user_id()))? + .having(|e| e.column(OrmUnitOrder::amount())?.gt(100))? + .finish() + }) + .await, Err(DatabaseError::AggMiss(_)) )); Ok(()) } #[test] - fn update_assignments_preserve_unmatched_rows() -> Result<(), DatabaseError> { - let database = build_orm_unit_database()?; + async fn update_assignments_preserve_unmatched_rows() -> Result<(), DatabaseError> { + let database = build_orm_unit_database().await?; assert!(matches!( - database.bind(|ctx| ctx.mutate::()?.update(|_| Ok(()))), + database + .bind(|ctx| ctx.mutate::()?.update(|_| Ok(()))) + .await, Err(DatabaseError::ColumnsEmpty) )); database @@ -4404,30 +4544,40 @@ mod tests { ScalarExpression::Constant(DataValue::Int32(25)), ) }) - })? - .done()?; + }) + .await? + .done() + .await?; let mut rows = database .bind(|ctx| { ctx.from::()? .project_scalars((OrmUnitUser::id(), OrmUnitUser::name(), OrmUnitUser::age()))? .order_by(OrmUnitUser::id().asc())? .finish() - })? + }) + .await? .project_tuple::<(i32, String, Option)>(); assert_eq!( - rows.next().transpose()?, + rows.next().await.transpose()?, Some((1, "Alicia".into(), Some(25))) ); - assert_eq!(rows.next().transpose()?, Some((2, "Bob".into(), Some(20)))); - assert_eq!(rows.next().transpose()?, Some((3, "Cara".into(), None))); - assert!(rows.next().transpose()?.is_none()); - rows.done()?; + assert_eq!( + rows.next().await.transpose()?, + Some((2, "Bob".into(), Some(20))) + ); + assert_eq!( + rows.next().await.transpose()?, + Some((3, "Cara".into(), None)) + ); + assert!(rows.next().await.transpose()?.is_none()); + rows.done().await?; Ok(()) } #[test] - fn query_builder_wrappers_cover_group_having_and_join_variants() -> Result<(), DatabaseError> { - let database = build_orm_unit_database()?; + async fn query_builder_wrappers_cover_group_having_and_join_variants( + ) -> Result<(), DatabaseError> { + let database = build_orm_unit_database().await?; let grouped_plan = database.explain(|ctx| { ctx.from::()? @@ -4586,8 +4736,8 @@ mod tests { } #[test] - fn query_builder_force_nested_loop_join() -> Result<(), DatabaseError> { - let database = build_orm_unit_database()?; + async fn query_builder_force_nested_loop_join() -> Result<(), DatabaseError> { + let database = build_orm_unit_database().await?; let plan = database.explain(|ctx| { ctx.from::()? @@ -4615,8 +4765,8 @@ mod tests { #[cfg(feature = "spill")] #[test] - fn query_builder_force_spill_aggregate_and_distinct() -> Result<(), DatabaseError> { - let database = build_orm_unit_database()?; + async fn query_builder_force_spill_aggregate_and_distinct() -> Result<(), DatabaseError> { + let database = build_orm_unit_database().await?; let plan = database.explain(|ctx| { ctx.from::()? @@ -4662,5 +4812,29 @@ mod tests { Ok(()) } + + #[async_only] + #[test] + async fn async_orm_futures_are_send() -> Result<(), DatabaseError> { + fn assert_send(_: F) {} + + let database = crate::db::DataBaseBuilder::path("./orm-unit-test") + .build_in_memory() + .await?; + assert_send(database.insert(&OrmUnitUser::default())); + assert_send(database.get::(&1)); + assert_send(database.bind(|ctx| ctx.mutate::()?.delete())); + assert_send(async { + database + .fetch::() + .await? + .next() + .await + .transpose() + }); + let mut tx = database.new_transaction(false)?; + assert_send(async { tx.fetch::().await?.next().await.transpose() }); + Ok(()) + } } // GRCOV_EXCL_STOP diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 98c76d3c..d7d0b55f 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -46,6 +46,7 @@ pub(crate) fn parse_sql_with_recursion_limit>( // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::parse_sql; use sqlparser::ast::{SetExpr, Statement}; diff --git a/src/planner/arena.rs b/src/planner/arena.rs index e6563f6f..de1612ab 100644 --- a/src/planner/arena.rs +++ b/src/planner/arena.rs @@ -18,6 +18,7 @@ use crate::db::Param; use crate::errors::DatabaseError; use crate::expression::visitor_mut::ExprVisitorMut; use crate::expression::ScalarExpression; +use crate::maybe_async::{MaybeSend, MaybeSync}; use crate::planner::operator::scalar_query_init::ScalarQueryInitOperator; use crate::planner::operator::visitor_mut::{ walk_mut_operator, ChildrenVisitMut, OperatorExprVisitorMut, OperatorVisitorMut, @@ -229,7 +230,7 @@ impl std::ops::DerefMut for ArenaExprMut<'_> { } } -pub trait MetaArena { +pub trait MetaArena: MaybeSend + MaybeSync { fn table_arena_cell<'a>(&self) -> &'a TableArenaCell where Self: 'a, @@ -1375,6 +1376,7 @@ impl MetaArena for ParamArena<'_> { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/planner/mod.rs b/src/planner/mod.rs index 24fc5ae2..b30989a3 100644 --- a/src/planner/mod.rs +++ b/src/planner/mod.rs @@ -559,6 +559,7 @@ impl crate::serdes::ReferenceSerialization for LogicalPlan { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef, TableCatalog}; @@ -1076,6 +1077,7 @@ mod tests { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] pub(crate) mod test { use crate::expression::ScalarExpression; use crate::planner::{ExprRef, PlanArena}; diff --git a/src/planner/operator/join.rs b/src/planner/operator/join.rs index 77412c6f..92ee4aae 100644 --- a/src/planner/operator/join.rs +++ b/src/planner/operator/join.rs @@ -266,6 +266,7 @@ pub(crate) fn extract_join_keys( // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::*; diff --git a/src/planner/operator/mod.rs b/src/planner/operator/mod.rs index 34e1c919..7b7c2658 100644 --- a/src/planner/operator/mod.rs +++ b/src/planner/operator/mod.rs @@ -1140,6 +1140,7 @@ impl Explain for PhysicalOption { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::view::View; diff --git a/src/planner/operator/visitor.rs b/src/planner/operator/visitor.rs index c772a66d..69e1cf2c 100644 --- a/src/planner/operator/visitor.rs +++ b/src/planner/operator/visitor.rs @@ -605,6 +605,7 @@ pub fn walk_operator<'a, V: OperatorVisitor<'a>>( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] pub(crate) mod tests { use super::*; #[cfg(feature = "copy")] diff --git a/src/planner/operator/visitor_mut.rs b/src/planner/operator/visitor_mut.rs index f7c17f9b..7b0d4c2f 100644 --- a/src/planner/operator/visitor_mut.rs +++ b/src/planner/operator/visitor_mut.rs @@ -632,6 +632,7 @@ pub fn walk_mut_operator<'a, V: OperatorVisitorMut<'a>>( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::planner::operator::visitor::tests::all_operators; diff --git a/src/planner/operator/window.rs b/src/planner/operator/window.rs index 4013ff38..16cfccea 100644 --- a/src/planner/operator/window.rs +++ b/src/planner/operator/window.rs @@ -82,13 +82,14 @@ impl Explain for WindowOperator { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::expression::window::WindowFunctionKind; use crate::expression::ScalarExpression; use crate::planner::{ExprRef, PlanArena, TableArena, TableArenaCell}; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::LogicalType; use std::io::{Cursor, Seek, SeekFrom}; @@ -162,7 +163,7 @@ mod tests { cursor.seek(SeekFrom::Start(0))?; assert_eq!( - WindowOperator::decode::( + WindowOperator::decode::( &mut cursor, None, &mut TableArena::default(), diff --git a/src/planner/plan_keeper.rs b/src/planner/plan_keeper.rs index 07b9bc39..c723bad7 100644 --- a/src/planner/plan_keeper.rs +++ b/src/planner/plan_keeper.rs @@ -35,6 +35,12 @@ impl<'p> PlanKeeper<'p> { } } +// SAFETY: `PlanKeeper` owns the plan like a `Box` and only lends `&LogicalPlan`; moving it to +// another thread moves the plan and every borrower with it, which needs `LogicalPlan: Send + Sync`. +unsafe impl Send for PlanKeeper<'_> where LogicalPlan: Send + Sync {} +// SAFETY: shared access only ever reads the plan. +unsafe impl Sync for PlanKeeper<'_> where LogicalPlan: Sync {} + impl Drop for PlanKeeper<'_> { fn drop(&mut self) { // SAFETY: `owned` comes from `Box::into_raw` and is only freed here. diff --git a/src/planner/scalar_query_ref.rs b/src/planner/scalar_query_ref.rs index bcf46734..6370fec0 100644 --- a/src/planner/scalar_query_ref.rs +++ b/src/planner/scalar_query_ref.rs @@ -61,10 +61,11 @@ impl ReferenceSerialization for ScalarQueryRef { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::planner::TableArena; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; #[test] fn decoding_preserves_source_reference() -> Result<(), DatabaseError> { @@ -73,21 +74,21 @@ mod tests { let mut bytes = Vec::new(); reference.encode(&mut bytes, false, &source)?; let mut target = TableArena::default(); - let mut context = ReferenceDecodeContext::::new(); - let first = ScalarQueryRef::decode::( + let mut context = ReferenceDecodeContext::::new(); + let first = ScalarQueryRef::decode::( &mut bytes.as_slice(), Some(&mut context), &mut target, )?; - let repeated = ScalarQueryRef::decode::( + let repeated = ScalarQueryRef::decode::( &mut bytes.as_slice(), Some(&mut context), &mut target, )?; assert_eq!(first, repeated); assert_eq!(first, reference); - let mut next_view = ReferenceDecodeContext::::new(); - let second = ScalarQueryRef::decode::( + let mut next_view = ReferenceDecodeContext::::new(); + let second = ScalarQueryRef::decode::( &mut bytes.as_slice(), Some(&mut next_view), &mut target, diff --git a/src/serdes/char_length_units.rs b/src/serdes/char_length_units.rs index 828de26a..0f4ef27c 100644 --- a/src/serdes/char_length_units.rs +++ b/src/serdes/char_length_units.rs @@ -53,10 +53,11 @@ impl ReferenceSerialization for CharLengthUnits { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] pub(crate) mod test { use crate::errors::DatabaseError; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::CharLengthUnits; use std::io::{Cursor, Seek, SeekFrom}; @@ -68,14 +69,14 @@ pub(crate) mod test { CharLengthUnits::Characters.encode(&mut cursor, false, &arena)?; cursor.seek(SeekFrom::Start(0))?; assert_eq!( - CharLengthUnits::decode::(&mut cursor, None, &mut arena,)?, + CharLengthUnits::decode::(&mut cursor, None, &mut arena,)?, CharLengthUnits::Characters ); cursor.seek(SeekFrom::Start(0))?; CharLengthUnits::Octets.encode(&mut cursor, false, &arena)?; cursor.seek(SeekFrom::Start(0))?; assert_eq!( - CharLengthUnits::decode::(&mut cursor, None, &mut arena,)?, + CharLengthUnits::decode::(&mut cursor, None, &mut arena,)?, CharLengthUnits::Octets ); diff --git a/src/serdes/column.rs b/src/serdes/column.rs index bfd65b92..819d5dff 100644 --- a/src/serdes/column.rs +++ b/src/serdes/column.rs @@ -124,6 +124,7 @@ impl ReferenceSerialization for ColumnRelation { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] pub(crate) mod test { use crate::catalog::{ ColumnCatalog, ColumnDesc, ColumnRef, ColumnRelation, ColumnSummary, TableName, @@ -133,18 +134,15 @@ pub(crate) mod test { use crate::expression::ScalarExpression; use crate::planner::{MetaArena, PlanArena, TableArenaCell}; use crate::serdes::{ReferenceDecodeContext, ReferenceSerialization}; - use crate::storage::rocksdb::RocksStorage; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::storage::{Storage, StorageTransaction}; use crate::types::value::DataValue; use crate::types::LogicalType; use std::io::{Cursor, Seek, SeekFrom}; - use tempfile::TempDir; #[test] - fn test_column_serialization() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_column_serialization() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let mut transaction = storage.transaction()?; let mut names = crate::storage::NameMapping::default(); let mut table_cache = crate::storage::TableCache::default(); @@ -157,7 +155,8 @@ pub(crate) mod test { &mut table_cache, &mut transaction, &mut plan_arena, - )?; + ) + .await?; let mut plan_arena = PlanArena::new(&table_arena); let mut cursor = Cursor::new(Vec::new()); @@ -173,7 +172,7 @@ pub(crate) mod test { assert_eq!( { let mut context = ReferenceDecodeContext::new(); - ColumnRef::decode::>, _>( + ColumnRef::decode::>, _>( &mut cursor, Some(&mut context), &mut plan_arena, @@ -184,14 +183,15 @@ pub(crate) mod test { cursor.seek(SeekFrom::Start(0))?; let mut table_codec = crate::storage::table_codec::TableCodec::default(); - let table = - transaction.drop_column(&mut table_codec, &mut plan_arena, &table_name, "c3")?; + let table = transaction + .drop_column(&mut table_codec, &mut plan_arena, &table_name, "c3") + .await?; let table = table.transplant_to_table_arena(&plan_arena)?; names.insert_table(&mut table_cache, table); plan_arena = PlanArena::new(&table_arena); let mut context = ReferenceDecodeContext::new(); assert_eq!( - ColumnRef::decode::>, _>( + ColumnRef::decode::>, _>( &mut cursor, Some(&mut context), &mut plan_arena, @@ -215,7 +215,7 @@ pub(crate) mod test { not_ref_column.encode(&mut cursor, false, &plan_arena)?; cursor.seek(SeekFrom::Start(0))?; - let decoded = ColumnRef::decode::>, _>( + let decoded = ColumnRef::decode::>, _>( &mut cursor, None, &mut plan_arena, @@ -250,7 +250,7 @@ pub(crate) mod test { cursor.seek(SeekFrom::Start(0))?; assert_eq!( - ColumnSummary::decode::>, _>( + ColumnSummary::decode::>, _>( &mut cursor, None, &mut arena, @@ -269,7 +269,7 @@ pub(crate) mod test { none_relation.encode(&mut cursor, false, &arena)?; cursor.seek(SeekFrom::Start(0))?; - let decode_relation = ColumnRelation::decode::>, _>( + let decode_relation = ColumnRelation::decode::>, _>( &mut cursor, None, &mut arena, @@ -283,7 +283,7 @@ pub(crate) mod test { table_relation.encode(&mut cursor, false, &arena)?; cursor.seek(SeekFrom::Start(0))?; - let decode_relation = ColumnRelation::decode::>, _>( + let decode_relation = ColumnRelation::decode::>, _>( &mut cursor, None, &mut arena, @@ -306,7 +306,7 @@ pub(crate) mod test { desc.encode(&mut cursor, false, &arena)?; cursor.seek(SeekFrom::Start(0))?; - let decode_desc = ColumnDesc::decode::>, _>( + let decode_desc = ColumnDesc::decode::>, _>( &mut cursor, None, &mut arena, diff --git a/src/serdes/data_value.rs b/src/serdes/data_value.rs index 51dda37d..04616e49 100644 --- a/src/serdes/data_value.rs +++ b/src/serdes/data_value.rs @@ -417,11 +417,12 @@ fn read_char_length_units(reader: &mut R) -> Result(&mut cursor, None, &mut arena)?; + DataValue::decode::(&mut cursor, None, &mut arena)?; assert_eq!(source, decoded); } @@ -555,7 +556,7 @@ pub(crate) mod test { for (index, value) in values.into_iter().enumerate() { let mut bytes = Vec::new(); value.encode(&mut bytes, false, &arena)?; - let decoded = Utf8Type::decode::( + let decoded = Utf8Type::decode::( &mut Cursor::new(bytes), None, &mut arena, @@ -569,7 +570,7 @@ pub(crate) mod test { } let result = - Utf8Type::decode::(&mut Cursor::new([2]), None, &mut arena); + Utf8Type::decode::(&mut Cursor::new([2]), None, &mut arena); let Err(err) = result else { panic!("invalid tag should fail") }; diff --git a/src/serdes/evaluator.rs b/src/serdes/evaluator.rs index c9cceec2..9a07f897 100644 --- a/src/serdes/evaluator.rs +++ b/src/serdes/evaluator.rs @@ -204,9 +204,10 @@ impl ReferenceSerialization for CastEvaluatorRef { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::CharLengthUnits; use std::io::{Cursor, Seek, SeekFrom}; @@ -222,7 +223,7 @@ mod tests { cursor.seek(SeekFrom::Start(0))?; assert_eq!( - CastEvaluatorParams::decode::(&mut cursor, None, &mut arena,)?, + CastEvaluatorParams::decode::(&mut cursor, None, &mut arena,)?, params ); @@ -237,7 +238,7 @@ mod tests { cursor.seek(SeekFrom::Start(0))?; assert_eq!( - CastEvaluatorRef::decode::(&mut cursor, None, &mut arena,)?, + CastEvaluatorRef::decode::(&mut cursor, None, &mut arena,)?, evaluator ); diff --git a/src/serdes/function.rs b/src/serdes/function.rs index af563223..da16a7ac 100644 --- a/src/serdes/function.rs +++ b/src/serdes/function.rs @@ -95,12 +95,13 @@ impl ReferenceSerialization for ArcTableFunctionImpl { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::db::{ScalaFunctions, TableFunctions}; use crate::planner::TableArena; use crate::serdes::ReferenceDecodeContext; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::LogicalType; use std::io::Cursor; @@ -121,7 +122,7 @@ mod tests { let bytes = encoded_summary("missing_scalar"); let mut arena = TableArena::default(); - let err = ArcScalarFunctionImpl::decode::( + let err = ArcScalarFunctionImpl::decode::( &mut Cursor::new(&bytes), None, &mut arena, @@ -134,7 +135,7 @@ mod tests { let scalars = ScalaFunctions::default(); let table_functions = TableFunctions::default(); let mut context = ReferenceDecodeContext::with_functions(&scalars, &table_functions); - let err = ArcScalarFunctionImpl::decode::( + let err = ArcScalarFunctionImpl::decode::( &mut Cursor::new(&bytes), Some(&mut context), &mut arena, @@ -148,7 +149,7 @@ mod tests { let bytes = encoded_summary("missing_table"); let mut arena = TableArena::default(); - let err = ArcTableFunctionImpl::decode::( + let err = ArcTableFunctionImpl::decode::( &mut Cursor::new(&bytes), None, &mut arena, @@ -161,7 +162,7 @@ mod tests { let scalars = ScalaFunctions::default(); let table_functions = TableFunctions::default(); let mut context = ReferenceDecodeContext::with_functions(&scalars, &table_functions); - let err = ArcTableFunctionImpl::decode::( + let err = ArcTableFunctionImpl::decode::( &mut Cursor::new(&bytes), Some(&mut context), &mut arena, diff --git a/src/serdes/mod.rs b/src/serdes/mod.rs index e0dd73dd..0e56334a 100644 --- a/src/serdes/mod.rs +++ b/src/serdes/mod.rs @@ -117,12 +117,13 @@ impl<'a, T: StorageTransaction> ReferenceDecodeContext<'a, T> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::errors::DatabaseError; use crate::expression::range_detacher::Range; use crate::planner::TableArena; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::value::DataValue; use std::collections::{BTreeMap, Bound}; use std::fmt::Debug; @@ -140,7 +141,7 @@ mod tests { source.encode(&mut cursor, false, &arena)?; cursor.seek(SeekFrom::Start(0))?; - S::decode::(&mut cursor, None, &mut arena) + S::decode::(&mut cursor, None, &mut arena) } #[test] diff --git a/src/serdes/num.rs b/src/serdes/num.rs index 8aa560fd..9898ae48 100644 --- a/src/serdes/num.rs +++ b/src/serdes/num.rs @@ -102,10 +102,11 @@ impl ReferenceSerialization for usize { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] pub(crate) mod test { use crate::errors::DatabaseError; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use std::io::{Cursor, Seek, SeekFrom}; #[test] @@ -140,28 +141,26 @@ pub(crate) mod test { cursor.seek(SeekFrom::Start(0))?; - let decoded_0 = - u8::decode::(&mut cursor, None, &mut arena).unwrap(); + let decoded_0 = u8::decode::(&mut cursor, None, &mut arena).unwrap(); let decoded_1 = - u16::decode::(&mut cursor, None, &mut arena).unwrap(); + u16::decode::(&mut cursor, None, &mut arena).unwrap(); let decoded_2 = - u32::decode::(&mut cursor, None, &mut arena).unwrap(); + u32::decode::(&mut cursor, None, &mut arena).unwrap(); let decoded_3 = - u64::decode::(&mut cursor, None, &mut arena).unwrap(); - let decoded_4 = - i8::decode::(&mut cursor, None, &mut arena).unwrap(); + u64::decode::(&mut cursor, None, &mut arena).unwrap(); + let decoded_4 = i8::decode::(&mut cursor, None, &mut arena).unwrap(); let decoded_5 = - i16::decode::(&mut cursor, None, &mut arena).unwrap(); + i16::decode::(&mut cursor, None, &mut arena).unwrap(); let decoded_6 = - i32::decode::(&mut cursor, None, &mut arena).unwrap(); + i32::decode::(&mut cursor, None, &mut arena).unwrap(); let decoded_7 = - i64::decode::(&mut cursor, None, &mut arena).unwrap(); + i64::decode::(&mut cursor, None, &mut arena).unwrap(); let decoded_8 = - f32::decode::(&mut cursor, None, &mut arena).unwrap(); + f32::decode::(&mut cursor, None, &mut arena).unwrap(); let decoded_9 = - f64::decode::(&mut cursor, None, &mut arena).unwrap(); + f64::decode::(&mut cursor, None, &mut arena).unwrap(); let decoded_10 = - usize::decode::(&mut cursor, None, &mut arena).unwrap(); + usize::decode::(&mut cursor, None, &mut arena).unwrap(); assert_eq!(source_0, decoded_0); assert_eq!(source_1, decoded_1); diff --git a/src/serdes/physical_option.rs b/src/serdes/physical_option.rs index 85628222..6f66f3c0 100644 --- a/src/serdes/physical_option.rs +++ b/src/serdes/physical_option.rs @@ -98,7 +98,7 @@ mod tests { #[cfg(all(not(feature = "rocksdb"), feature = "lmdb"))] use crate::storage::lmdb::LmdbTransaction as TestTransaction; #[cfg(feature = "rocksdb")] - use crate::storage::rocksdb::OptimisticRocksTransaction as TestTransaction; + use crate::storage::TestTransaction; use crate::types::value::DataValue; #[test] @@ -130,11 +130,8 @@ mod tests { let mut encoded = Vec::new(); option.encode(&mut encoded, is_direct, &arena)?; let mut reader = encoded.as_slice(); - let decoded = PhysicalOption::decode::, _, _>( - &mut reader, - None, - &mut arena, - )?; + let decoded = + PhysicalOption::decode::(&mut reader, None, &mut arena)?; assert_eq!(decoded, option); assert!(reader.is_empty()); } @@ -179,7 +176,7 @@ mod tests { assert_eq!(encoded, expected); let mut reader = encoded.as_slice(); let decoded = - PhysicalOption::decode::, _, _>(&mut reader, None, &mut arena)?; + PhysicalOption::decode::(&mut reader, None, &mut arena)?; assert_eq!(decoded.plan, PlanImpl::Sort); assert_eq!( decoded.row_count, diff --git a/src/serdes/stable_hash.rs b/src/serdes/stable_hash.rs index 5fd1b0a2..02b13431 100644 --- a/src/serdes/stable_hash.rs +++ b/src/serdes/stable_hash.rs @@ -115,6 +115,7 @@ fn mix64(mut value: u64) -> u64 { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::StableHasher; use std::hash::{Hash, Hasher}; diff --git a/src/serdes/string.rs b/src/serdes/string.rs index d39f5e16..1e7f16a2 100644 --- a/src/serdes/string.rs +++ b/src/serdes/string.rs @@ -70,10 +70,11 @@ impl ReferenceSerialization for Arc { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] pub(crate) mod test { use crate::errors::DatabaseError; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use std::io::{Cursor, Seek, SeekFrom}; #[test] @@ -86,7 +87,7 @@ pub(crate) mod test { ReferenceSerialization::encode(&source, &mut cursor, true, &arena)?; cursor.seek(SeekFrom::Start(0))?; assert_eq!( - String::decode::(&mut cursor, None, &mut arena,)?, + String::decode::(&mut cursor, None, &mut arena,)?, source ); diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 419fb0ab..d14f3b6b 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -14,9 +14,12 @@ #[cfg(all(not(target_arch = "wasm32"), feature = "lmdb"))] pub mod lmdb; +#[cfg(not(feature = "async"))] pub mod memory; #[cfg(all(not(target_arch = "wasm32"), feature = "rocksdb"))] pub mod rocksdb; +#[cfg(feature = "slatedb")] +pub mod slatedb; pub(crate) mod table_codec; use crate::catalog::view::View; @@ -26,6 +29,7 @@ use crate::errors::DatabaseError; use crate::expression::range_detacher::{Range, RangeValue}; use crate::expression::TypeCast; use crate::iter_ext::Itertools; +use crate::maybe_async::{MaybeSend, MaybeStatic, MaybeSync}; use crate::optimizer::core::cm_sketch::{ CountMinSketch, CountMinSketchPage, COUNT_MIN_SKETCH_STORAGE_PAGE_LEN, }; @@ -157,7 +161,7 @@ impl Display for EmptyStorageMetrics { } } -pub trait Storage: Clone { +pub trait Storage: Clone + MaybeSend + MaybeSync { type Metrics: Display; type TransactionType<'a>: StorageTransaction @@ -205,8 +209,9 @@ pub trait CheckpointableStorage: Storage { /// Optional bounds of the reader, of the form (offset, limit). pub(crate) type Bounds = (Option, Option); -pub trait StorageTransaction: Sized { - type BorrowedBytes<'a>: AsRef<[u8]> +#[maybe_async] +pub trait StorageTransaction: Sized + MaybeSend + MaybeSync + MaybeStatic { + type BorrowedBytes<'a>: AsRef<[u8]> + MaybeSend where Self: 'a; @@ -218,7 +223,7 @@ pub trait StorageTransaction: Sized { where Self: 'a; - type PointCursor<'a> + type PointCursor<'a>: MaybeSend + MaybeSync where Self: 'a; @@ -233,7 +238,7 @@ pub trait StorageTransaction: Sized { } /// The bounds is applied to the whole data batches, not per batch. - fn read<'a>( + async fn read<'a>( &'a self, table_codec: &mut TableCodec, deserializers: &'a TupleDeserializers, @@ -242,17 +247,21 @@ pub trait StorageTransaction: Sized { ) -> Result, DatabaseError> { let offset = bounds.0.unwrap_or(0); - table_codec.with_tuple_bound(table_id, |min, max, pool| { - let iter = self.range(Bound::Included(min), Bound::Included(max), pool)?; - - Ok(TupleIter { - table_id, - bounds: IterBounds::new(offset, bounds.1), - deserializers, - iter, - mapped: self.mapped_scope(), + table_codec + .with_tuple_bound(table_id, async |min, max, pool| { + let iter = self + .range(Bound::Included(min), Bound::Included(max), pool) + .await?; + + Ok(TupleIter { + table_id, + bounds: IterBounds::new(offset, bounds.1), + deserializers, + iter, + mapped: self.mapped_scope(), + }) }) - }) + .await } #[allow(clippy::too_many_arguments)] @@ -287,7 +296,7 @@ pub trait StorageTransaction: Sized { }) } - fn add_index_meta( + async fn add_index_meta( &mut self, table_codec: &mut TableCodec, plan_arena: &mut (dyn MetaArena + '_), @@ -297,7 +306,8 @@ pub trait StorageTransaction: Sized { ty: IndexType, ) -> Result<(TableCatalog, IndexId), DatabaseError> { let mut table = self - .load_table(table_codec, plan_arena, table_name)? + .load_table(table_codec, plan_arena, table_name) + .await? .ok_or(DatabaseError::TableNotFound)?; let index_meta = table.add_index_meta(index_name, column_ids, ty, plan_arena)?; let index_meta = plan_arena.index(index_meta); @@ -313,7 +323,7 @@ pub trait StorageTransaction: Sized { Ok((table, index_id)) } - fn add_index( + async fn add_index( &mut self, table_codec: &mut TableCodec, table_id: TableId, @@ -323,23 +333,25 @@ pub trait StorageTransaction: Sized { if matches!(index.ty, IndexType::PrimaryKey { .. }) { return Ok(()); } - table_codec.with_index(table_id, &index, Some(tuple_id), |key, value| { - if matches!(index.ty, IndexType::Unique) { - if let Some(bytes) = self.get_borrowed(key)? { - return if TableCodec::strip_stamp(bytes.as_ref())? - != TableCodec::strip_stamp(value)? - { - Err(DatabaseError::DuplicateUniqueValue) - } else { - Ok(()) - }; + table_codec + .with_index(table_id, &index, Some(tuple_id), async |key, value| { + if matches!(index.ty, IndexType::Unique) { + if let Some(bytes) = self.get_borrowed(key).await? { + return if TableCodec::strip_stamp(bytes.as_ref())? + != TableCodec::strip_stamp(value)? + { + Err(DatabaseError::DuplicateUniqueValue) + } else { + Ok(()) + }; + } } - } - self.set(key, value) - }) + self.set(key, value) + }) + .await } - fn del_index( + async fn del_index( &mut self, table_codec: &mut TableCodec, table_id: TableId, @@ -349,12 +361,16 @@ pub trait StorageTransaction: Sized { if matches!(index.ty, IndexType::PrimaryKey { .. }) { return Ok(()); } - table_codec.with_index(table_id, index, Some(tuple_id), |key, _| self.remove(key))?; + table_codec + .with_index(table_id, index, Some(tuple_id), async |key, _| { + self.remove(key) + }) + .await?; Ok(()) } - fn append_tuple( + async fn append_tuple( &mut self, table_codec: &mut TableCodec, table_id: TableId, @@ -363,48 +379,59 @@ pub trait StorageTransaction: Sized { is_overwrite: bool, ) -> Result<(), DatabaseError> where - I: IntoIterator, + I: IntoIterator + MaybeSend, + I::IntoIter: MaybeSend, S: Borrow, { let tuple_id = tuple.pk().ok_or(DatabaseError::PrimaryKeyNotFound)?; let mut serializers = serializers.into_iter(); let mut write_value = |value: &mut Bytes| tuple.serialize_to(&mut serializers, value); - table_codec.with_tuple( - table_id, - tuple_id, - Some((tuple.serialized_len_hint(), &mut write_value)), - |key, value| { - if !is_overwrite && self.exists(key)? { - return Err(DatabaseError::DuplicatePrimaryKey); - } - self.set(key, value) - }, - ) + table_codec + .with_tuple( + table_id, + tuple_id, + Some((tuple.serialized_len_hint(), &mut write_value)), + async |key, value| { + if !is_overwrite && self.exists(key).await? { + return Err(DatabaseError::DuplicatePrimaryKey); + } + self.set(key, value) + }, + ) + .await } - fn remove_tuple( + async fn remove_tuple( &mut self, table_codec: &mut TableCodec, table_id: TableId, tuple_id: &TupleId, ) -> Result<(), DatabaseError> { - table_codec.with_tuple(table_id, tuple_id, None, |key, _| self.remove(key)) + table_codec + .with_tuple(table_id, tuple_id, None, async |key, _| self.remove(key)) + .await } - fn rewrite_table_metadata( + async fn rewrite_table_metadata( &mut self, table_codec: &mut TableCodec, arena: &(impl MetaArena + ?Sized), table: &TableCatalog, ) -> Result<(), DatabaseError> { let table_id = table.id(); - table_codec.with_columns_bound(table_id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + table_codec + .with_columns_bound(table_id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - table_codec.with_index_meta_bound(table_id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + table_codec + .with_index_meta_bound(table_id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; for column in table.columns().map(|column| arena.column(*column)) { table_codec.with_column(table_id, column, true, arena, |key, value| { @@ -425,7 +452,7 @@ pub trait StorageTransaction: Sized { } #[allow(clippy::too_many_arguments)] - fn change_column( + async fn change_column( &mut self, table_codec: &mut TableCodec, plan_arena: &mut (dyn MetaArena + '_), @@ -437,7 +464,8 @@ pub trait StorageTransaction: Sized { not_null_change: &NotNullChange, ) -> Result { let table = self - .load_table(table_codec, plan_arena, table_name)? + .load_table(table_codec, plan_arena, table_name) + .await? .ok_or(DatabaseError::TableNotFound)?; let mut column_catalogs = Vec::with_capacity(table.columns_len()); let mut found = false; @@ -513,15 +541,19 @@ pub trait StorageTransaction: Sized { index_metas.into_iter(), plan_arena, )?; - self.rewrite_table_metadata(table_codec, plan_arena, &updated_table)?; - table_codec.with_statistics_bound(updated_table.id(), |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + self.rewrite_table_metadata(table_codec, plan_arena, &updated_table) + .await?; + table_codec + .with_statistics_bound(updated_table.id(), async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; Ok(updated_table) } - fn add_column( + async fn add_column( &mut self, table_codec: &mut TableCodec, plan_arena: &mut (dyn MetaArena + '_), @@ -530,7 +562,8 @@ pub trait StorageTransaction: Sized { if_not_exists: bool, ) -> Result<(TableCatalog, ColumnId), DatabaseError> { let mut table = self - .load_table(table_codec, plan_arena, table_name)? + .load_table(table_codec, plan_arena, table_name) + .await? .ok_or(DatabaseError::TableNotFound)?; if !column.nullable() && column.default_value(plan_arena)?.is_none() { return Err(DatabaseError::NeedNullAbleOrDefault); @@ -572,7 +605,7 @@ pub trait StorageTransaction: Sized { Ok((table, col_id)) } - fn drop_column( + async fn drop_column( &mut self, table_codec: &mut TableCodec, plan_arena: &mut (dyn MetaArena + '_), @@ -580,7 +613,8 @@ pub trait StorageTransaction: Sized { column_name: &str, ) -> Result { let table_catalog = self - .load_table(table_codec, plan_arena, table_name)? + .load_table(table_codec, plan_arena, table_name) + .await? .ok_or(DatabaseError::TableNotFound)?; let column_id = { let column = plan_arena.column(table_catalog.get_column_by_name(column_name).unwrap()); @@ -592,7 +626,8 @@ pub trait StorageTransaction: Sized { }; let table_id = table_catalog.id(); - self.remove_column_statistics(table_codec, table_id, column_id, plan_arena)?; + self.remove_column_statistics(table_codec, table_id, column_id, plan_arena) + .await?; for index_meta in table_catalog.indexes.iter() { let index_meta = plan_arena.index(*index_meta); if !index_meta.column_ids.contains(&column_id) { @@ -602,35 +637,43 @@ pub trait StorageTransaction: Sized { self.remove(key) })?; - table_codec.with_index_bound(table_id, index_meta.id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + table_codec + .with_index_bound(table_id, index_meta.id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - self.remove_index_statistics(table_codec, table_id, index_meta.id)?; + self.remove_index_statistics(table_codec, table_id, index_meta.id) + .await?; } - self.load_table(table_codec, plan_arena, table_name)? + self.load_table(table_codec, plan_arena, table_name) + .await? .ok_or(DatabaseError::TableNotFound) } - fn create_view( + async fn create_view( &mut self, table_codec: &mut TableCodec, arena: &mut (dyn MetaArena + '_), view: &View, or_replace: bool, ) -> Result { - let view_id = match self.root_table(table_codec, &view.name, arena)? { + let view_id = match self.root_table(table_codec, &view.name, arena).await? { Some(meta) if !meta.is_view => return Err(DatabaseError::TableExists), Some(_) if !or_replace => return Err(DatabaseError::ViewExists), Some(meta) => meta.id, - None => self.create_root_table(table_codec, arena, view.name.clone(), true)?, + None => { + self.create_root_table(table_codec, arena, view.name.clone(), true) + .await? + } }; table_codec.with_view_value(view_id, view, arena, |key, value| self.set(key, value))?; Ok(view_id) } - fn create_table( + async fn create_table( &mut self, table_codec: &mut TableCodec, plan_arena: &mut (dyn MetaArena + '_), @@ -638,14 +681,18 @@ pub trait StorageTransaction: Sized { columns: Vec, if_not_exists: bool, ) -> Result, DatabaseError> { - match self.root_table(table_codec, &table_name, plan_arena)? { + match self + .root_table(table_codec, &table_name, plan_arena) + .await? + { Some(meta) if meta.is_view => return Err(DatabaseError::ViewExists), Some(_) if if_not_exists => return Ok(None), Some(_) => return Err(DatabaseError::TableExists), None => (), } - let table_id = - self.create_root_table(table_codec, plan_arena, table_name.clone(), false)?; + let table_id = self + .create_root_table(table_codec, plan_arena, table_name.clone(), false) + .await?; let mut table_catalog = TableCatalog::new(table_id, table_name, columns, plan_arena)?; for (_, column) in table_catalog.primary_keys() { @@ -666,48 +713,54 @@ pub trait StorageTransaction: Sized { } /// The table or view registered under `table_name`. - fn root_table( + async fn root_table( &self, table_codec: &mut TableCodec, table_name: &str, arena: &mut (impl MetaArena + ?Sized), ) -> Result, DatabaseError> { let bytes = table_codec - .with_root_table(table_name, None, arena, |key, _| self.get_borrowed(key))?; + .with_root_table(table_name, None, arena, async |key, _| { + self.get_borrowed(key).await + }) + .await?; bytes .map(|bytes| TableCodec::decode_root_table::(bytes.as_ref(), arena)) .transpose() } /// Registers a new table or view under `table_name` with the next id. - fn create_root_table( + async fn create_root_table( &mut self, table_codec: &mut TableCodec, arena: &(impl MetaArena + ?Sized), table_name: TableName, is_view: bool, ) -> Result { - let table_id = table_codec.with_next_table_id(|key| { - let table_id = match self.get_borrowed(key)? { - Some(bytes) => TableId::from_le_bytes( - bytes - .as_ref() - .try_into() - .map_err(|_| DatabaseError::InvalidValue("next table id".to_string()))?, - ), - None => 0, - }; - self.set(key, &(table_id + 1).to_le_bytes())?; - Ok(table_id) - })?; + let table_id = table_codec + .with_next_table_id(async |key| { + let table_id = match self.get_borrowed(key).await? { + Some(bytes) => { + TableId::from_le_bytes(bytes.as_ref().try_into().map_err(|_| { + DatabaseError::InvalidValue("next table id".to_string()) + })?) + } + None => 0, + }; + self.set(key, &(table_id + 1).to_le_bytes())?; + Ok(table_id) + }) + .await?; let meta = TableMeta::new(table_name.clone(), table_id, is_view); - table_codec.with_root_table(&table_name, Some(&meta), arena, |key, value| { - self.set(key, value) - })?; + table_codec + .with_root_table(&table_name, Some(&meta), arena, async |key, value| { + self.set(key, value) + }) + .await?; Ok(table_id) } - fn drop_view( + async fn drop_view( &mut self, table_codec: &mut TableCodec, arena: &mut (dyn MetaArena + '_), @@ -715,7 +768,8 @@ pub trait StorageTransaction: Sized { if_exists: bool, ) -> Result { let Some(meta) = self - .root_table(table_codec, &view_name, arena)? + .root_table(table_codec, &view_name, arena) + .await? .filter(|meta| meta.is_view) else { if if_exists { @@ -725,13 +779,17 @@ pub trait StorageTransaction: Sized { } }; - table_codec.with_view(meta.id, |key, _| self.remove(key))?; - table_codec.with_root_table(&view_name, None, arena, |key, _| self.remove(key))?; + table_codec + .with_view(meta.id, async |key, _| self.remove(key)) + .await?; + table_codec + .with_root_table(&view_name, None, arena, async |key, _| self.remove(key)) + .await?; Ok(true) } - fn drop_index( + async fn drop_index( &mut self, table_codec: &mut TableCodec, plan_arena: &mut (dyn MetaArena + '_), @@ -740,7 +798,8 @@ pub trait StorageTransaction: Sized { if_exists: bool, ) -> Result, DatabaseError> { let table = self - .load_table(table_codec, plan_arena, table_name)? + .load_table(table_codec, plan_arena, table_name) + .await? .ok_or(DatabaseError::TableNotFound)?; let Some(index_meta_ref) = table .indexes @@ -765,19 +824,24 @@ pub trait StorageTransaction: Sized { self.remove(key) })?; - table_codec.with_index_bound(table.id(), index_id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + table_codec + .with_index_bound(table.id(), index_id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - self.remove_index_statistics(table_codec, table.id(), index_id)?; + self.remove_index_statistics(table_codec, table.id(), index_id) + .await?; - self.load_table(table_codec, plan_arena, table_name)? + self.load_table(table_codec, plan_arena, table_name) + .await? .map(|table| (table, index_id)) .map(Some) .ok_or(DatabaseError::TableNotFound) } - fn drop_table( + async fn drop_table( &mut self, table_codec: &mut TableCodec, plan_arena: &mut (dyn MetaArena + '_), @@ -785,7 +849,8 @@ pub trait StorageTransaction: Sized { if_exists: bool, ) -> Result { let Some(meta) = self - .root_table(table_codec, &table_name, plan_arena)? + .root_table(table_codec, &table_name, plan_arena) + .await? .filter(|meta| !meta.is_view) else { if if_exists { @@ -794,39 +859,56 @@ pub trait StorageTransaction: Sized { return Err(DatabaseError::TableNotFound); } }; - self.drop_data(table_codec, meta.id)?; + self.drop_data(table_codec, meta.id).await?; - table_codec.with_columns_bound(meta.id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + table_codec + .with_columns_bound(meta.id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - table_codec.with_index_meta_bound(meta.id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + table_codec + .with_index_meta_bound(meta.id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - table_codec.with_root_table(table_name.as_ref(), None, plan_arena, |key, _| { - self.remove(key) - })?; + table_codec + .with_root_table(table_name.as_ref(), None, plan_arena, async |key, _| { + self.remove(key) + }) + .await?; Ok(true) } - fn drop_data( + async fn drop_data( &mut self, table_codec: &mut TableCodec, table_id: TableId, ) -> Result<(), DatabaseError> { - table_codec.with_tuple_bound(table_id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + table_codec + .with_tuple_bound(table_id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - table_codec.with_all_index_bound(table_id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + table_codec + .with_all_index_bound(table_id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - table_codec.with_statistics_bound(table_id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - }) + table_codec + .with_statistics_bound(table_id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await } #[allow(clippy::too_many_arguments)] @@ -843,7 +925,7 @@ pub trait StorageTransaction: Sized { Ok(names.view(view_cache, view_name)) } - fn load_view( + async fn load_view( &self, table_codec: &mut TableCodec, table_arena: &TableArenaCell, @@ -852,34 +934,41 @@ pub trait StorageTransaction: Sized { view_name: TableName, ) -> Result, DatabaseError> { let Some(meta) = self - .root_table(table_codec, &view_name, &mut *table_arena.borrow_mut())? + .root_table(table_codec, &view_name, &mut *table_arena.borrow_mut()) + .await? .filter(|meta| meta.is_view) else { return Ok(None); }; - table_codec.with_view(meta.id, |key, _| { - let Some(bytes) = self.get_borrowed(key)? else { - return Ok(None); - }; - TableCodec::decode_view::( - bytes.as_ref(), - scala_functions, - table_functions, - table_arena.borrow_mut(), - ) - .map(|view| Some((meta.id, view))) - }) + table_codec + .with_view(meta.id, async |key, _| { + let Some(bytes) = self.get_borrowed(key).await? else { + return Ok(None); + }; + TableCodec::decode_view::( + bytes.as_ref(), + scala_functions, + table_functions, + table_arena.borrow_mut(), + ) + .map(|view| Some((meta.id, view))) + }) + .await } - fn views<'a>( + async fn views<'a>( &'a self, table_codec: &mut TableCodec, ) -> Result, DatabaseError> { - table_codec.with_view_bound(|min, max, pool| { - Ok(ViewIter { - iter: self.range(Bound::Included(min), Bound::Included(max), pool)?, + table_codec + .with_view_bound(async |min, max, pool| { + Ok(ViewIter { + iter: self + .range(Bound::Included(min), Bound::Included(max), pool) + .await?, + }) }) - }) + .await } fn table<'a>( @@ -891,19 +980,21 @@ pub trait StorageTransaction: Sized { Ok(names.table(table_cache, table_name)) } - fn load_table( + async fn load_table( &self, table_codec: &mut TableCodec, arena: &mut (impl MetaArena + ?Sized), table_name: &str, ) -> Result, DatabaseError> { let Some(meta) = self - .root_table(table_codec, table_name, arena)? + .root_table(table_codec, table_name, arena) + .await? .filter(|meta| !meta.is_view) else { return Ok(None); }; - self.table_collect(table_codec, meta.id, arena)? + self.table_collect(table_codec, meta.id, arena) + .await? .map(|(columns, indexes)| { TableCatalog::reload( meta.id, @@ -916,18 +1007,22 @@ pub trait StorageTransaction: Sized { .transpose() } - fn tables<'a>( + async fn tables<'a>( &'a self, table_codec: &mut TableCodec, ) -> Result, DatabaseError> { - table_codec.with_root_table_bound(|min, max, pool| { - Ok(TableIter { - iter: self.range(Bound::Included(min), Bound::Included(max), pool)?, + table_codec + .with_root_table_bound(async |min, max, pool| { + Ok(TableIter { + iter: self + .range(Bound::Included(min), Bound::Included(max), pool) + .await?, + }) }) - }) + .await } - fn save_column_statistics( + async fn save_column_statistics( &mut self, table_codec: &mut TableCodec, table_id: TableId, @@ -935,28 +1030,35 @@ pub trait StorageTransaction: Sized { meta: &HistogramMeta, arena: &(impl MetaArena + ?Sized), ) -> Result<(), DatabaseError> { - table_codec.with_column_statistics(table_id, column_id, Some(meta), arena, |key, value| { - self.set(key, value) - }) + table_codec + .with_column_statistics( + table_id, + column_id, + Some(meta), + arena, + async |key, value| self.set(key, value), + ) + .await } - fn column_statistics( + async fn column_statistics( &self, table_codec: &mut TableCodec, table_id: TableId, column_id: ColumnId, arena: &mut (impl MetaArena + ?Sized), ) -> Result, DatabaseError> { - let value = - table_codec.with_column_statistics(table_id, column_id, None, arena, |key, _| { - self.get_borrowed(key) - })?; + let value = table_codec + .with_column_statistics(table_id, column_id, None, arena, async |key, _| { + self.get_borrowed(key).await + }) + .await?; value .map(|bytes| TableCodec::decode_column_statistics::(bytes.as_ref(), arena)) .transpose() } - fn remove_column_statistics( + async fn remove_column_statistics( &mut self, table_codec: &mut TableCodec, table_id: TableId, @@ -964,7 +1066,10 @@ pub trait StorageTransaction: Sized { arena: &(impl MetaArena + ?Sized), ) -> Result<(), DatabaseError> { table_codec - .with_column_statistics(table_id, column_id, None, arena, |key, _| self.remove(key)) + .with_column_statistics(table_id, column_id, None, arena, async |key, _| { + self.remove(key) + }) + .await } fn save_index_statistics( @@ -1024,107 +1129,117 @@ pub trait StorageTransaction: Sized { Ok(()) } - fn index_statistics( + async fn index_statistics( &self, table_codec: &mut TableCodec, table_id: TableId, index_id: IndexId, arena: &mut (impl MetaArena + ?Sized), ) -> Result, DatabaseError> { - table_codec.with_index_statistics_bound(table_id, index_id, |min, max, pool| { - let mut iter = self.range(Bound::Included(min), Bound::Included(max), pool)?; - let mut root = None; - let mut buckets = Vec::new(); - let mut sketch_meta = None; - let mut sketch_pages = Vec::::new(); - let mut top_n = None; - - while let Some((key, value)) = iter.try_next()? { - match TableCodec::decode_index_statistics_codec_type(key)? { - IndexStatisticsCodecType::Root => { - root = Some(TableCodec::decode_index_statistics::(value, arena)?); - } - IndexStatisticsCodecType::SketchMeta => { - sketch_meta = Some( - TableCodec::decode_index_statistics_sketch_meta::(value, arena)?, - ); - } - IndexStatisticsCodecType::SketchPage => { - sketch_pages.push(TableCodec::decode_index_statistics_sketch_page::( - value, arena, - )?); - } - IndexStatisticsCodecType::Bucket => { - buckets.push(TableCodec::decode_index_statistics_bucket::( - value, arena, - )?); - } - IndexStatisticsCodecType::TopN => { - top_n = Some(TableCodec::decode_index_statistics_top_n::( - value, arena, - )?); + table_codec + .with_index_statistics_bound(table_id, index_id, async |min, max, pool| { + let mut iter = self + .range(Bound::Included(min), Bound::Included(max), pool) + .await?; + let mut root = None; + let mut buckets = Vec::new(); + let mut sketch_meta = None; + let mut sketch_pages = Vec::::new(); + let mut top_n = None; + + while let Some((key, value)) = iter.try_next().await? { + match TableCodec::decode_index_statistics_codec_type(key)? { + IndexStatisticsCodecType::Root => { + root = Some(TableCodec::decode_index_statistics::(value, arena)?); + } + IndexStatisticsCodecType::SketchMeta => { + sketch_meta = Some(TableCodec::decode_index_statistics_sketch_meta::< + Self, + >(value, arena)?); + } + IndexStatisticsCodecType::SketchPage => { + sketch_pages.push(TableCodec::decode_index_statistics_sketch_page::< + Self, + >(value, arena)?); + } + IndexStatisticsCodecType::Bucket => { + buckets.push(TableCodec::decode_index_statistics_bucket::( + value, arena, + )?); + } + IndexStatisticsCodecType::TopN => { + top_n = Some(TableCodec::decode_index_statistics_top_n::( + value, arena, + )?); + } } } - } - iter.recycle(pool); + iter.recycle(pool); - match (root, sketch_meta) { - (Some(root), Some(sketch_meta)) => { - let sketch = CountMinSketch::from_storage_parts(sketch_meta, sketch_pages)?; - IndexStatistics::from_parts(root, buckets, sketch, top_n).map(Some) - } - (None, None) - if buckets.is_empty() && sketch_pages.is_empty() && top_n.is_none() => - { - Ok(None) + match (root, sketch_meta) { + (Some(root), Some(sketch_meta)) => { + let sketch = CountMinSketch::from_storage_parts(sketch_meta, sketch_pages)?; + IndexStatistics::from_parts(root, buckets, sketch, top_n).map(Some) + } + (None, None) + if buckets.is_empty() && sketch_pages.is_empty() && top_n.is_none() => + { + Ok(None) + } + _ => Err(DatabaseError::InvalidValue( + "statistics meta is incomplete".to_string(), + )), } - _ => Err(DatabaseError::InvalidValue( - "statistics meta is incomplete".to_string(), - )), - } - }) + }) + .await } - fn remove_index_statistics( + async fn remove_index_statistics( &mut self, table_codec: &mut TableCodec, table_id: TableId, index_id: IndexId, ) -> Result<(), DatabaseError> { - table_codec.with_index_statistics_bound(table_id, index_id, |min, max, pool| { - self.remove_range(Bound::Included(min), Bound::Included(max), pool) - })?; + table_codec + .with_index_statistics_bound(table_id, index_id, async |min, max, pool| { + self.remove_range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; Ok(()) } #[allow(clippy::type_complexity)] - fn table_collect( + async fn table_collect( &self, table_codec: &mut TableCodec, table_id: TableId, arena: &mut (impl MetaArena + ?Sized), ) -> Result, Vec)>, DatabaseError> { - table_codec.with_table_bound(table_id, |table_min, table_max, pool| { - let mut column_iter = - self.range(Bound::Included(table_min), Bound::Included(table_max), pool)?; - - let mut columns = Vec::new(); - let mut index_metas = Vec::new(); - - // Tips: only `Column`, `IndexMeta`, `TableMeta` - while let Some((key, value)) = column_iter.try_next().ok().flatten() { - if key.starts_with(table_min) { - let mut cursor = Cursor::new(value); - columns.push(TableCodec::decode_column::(&mut cursor, arena)?); - } else { - index_metas.push(TableCodec::decode_index_meta::(value, arena)?); + table_codec + .with_table_bound(table_id, async |table_min, table_max, pool| { + let mut column_iter = self + .range(Bound::Included(table_min), Bound::Included(table_max), pool) + .await?; + + let mut columns = Vec::new(); + let mut index_metas = Vec::new(); + + // Tips: only `Column`, `IndexMeta`, `TableMeta` + while let Some((key, value)) = column_iter.try_next().await.ok().flatten() { + if key.starts_with(table_min) { + let mut cursor = Cursor::new(value); + columns.push(TableCodec::decode_column::(&mut cursor, arena)?); + } else { + index_metas.push(TableCodec::decode_index_meta::(value, arena)?); + } } - } - column_iter.recycle(pool); + column_iter.recycle(pool); - Ok((!columns.is_empty()).then_some((columns, index_metas))) - }) + Ok((!columns.is_empty()).then_some((columns, index_metas))) + }) + .await } fn create_index_meta_from_column( @@ -1175,44 +1290,44 @@ pub trait StorageTransaction: Sized { Ok(()) } - fn get_borrowed<'a>( + async fn get_borrowed<'a>( &'a self, key: &[u8], ) -> Result>, DatabaseError>; fn point_cursor(&self) -> Result, DatabaseError>; - fn get_at<'a>( + async fn get_at<'a>( &'a self, _cursor: &Self::PointCursor<'a>, key: &[u8], ) -> Result>, DatabaseError> { - self.get_borrowed(key) + self.get_borrowed(key).await } - fn exists(&self, key: &[u8]) -> Result { - Ok(self.get_borrowed(key)?.is_some()) + async fn exists(&self, key: &[u8]) -> Result { + Ok(self.get_borrowed(key).await?.is_some()) } fn set(&mut self, key: &[u8], value: &[u8]) -> Result<(), DatabaseError>; fn remove(&mut self, key: &[u8]) -> Result<(), DatabaseError>; - fn range<'txn, 'key>( + async fn range<'txn, 'key>( &'txn self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, pool: &mut BytesPool, ) -> Result, DatabaseError>; - fn range_rev<'txn, 'key>( + async fn range_rev<'txn, 'key>( &'txn self, min: Bound<&'key [u8]>, max: Bound<&'key [u8]>, pool: &mut BytesPool, ) -> Result, DatabaseError>; - fn remove_range( + async fn remove_range( &mut self, min: Bound<&[u8]>, max: Bound<&[u8]>, @@ -1226,14 +1341,16 @@ pub trait StorageTransaction: Sized { loop { data_keys.clear(); - let mut iter = self.range( - bytes_bound_as_slice(&lower), - bytes_bound_as_slice(&upper), - pool, - )?; + let mut iter = self + .range( + bytes_bound_as_slice(&lower), + bytes_bound_as_slice(&upper), + pool, + ) + .await?; while data_keys.len() < DELETE_BATCH_SIZE { - let Some((key, _)) = iter.try_next()? else { + let Some((key, _)) = iter.try_next().await? else { break; }; data_keys.push(key.to_vec()); @@ -1257,7 +1374,7 @@ pub trait StorageTransaction: Sized { } } - fn commit(self) -> Result<(), DatabaseError>; + async fn commit(self) -> Result<(), DatabaseError>; } pub(crate) fn owned_bound(bound: Bound<&[u8]>) -> Bound { @@ -1309,8 +1426,9 @@ fn encode_bound_key(buffer: &mut Bytes, key: &[u8], is_upper: bool) { } } +#[maybe_async] #[inline] -fn encode_bound<'a>( +async fn encode_bound<'a>( table_codec: &mut TableCodec, bound: &Bound, is_upper: bool, @@ -1320,19 +1438,24 @@ fn encode_bound<'a>( ) -> Result, DatabaseError> { match bound { Bound::Included(val) => { - inner.bound_key(table_codec, params, val.value()?, is_upper, buffer)?; + inner + .bound_key(table_codec, params, val.value()?, is_upper, buffer) + .await?; Ok(Bound::Included(buffer.as_slice())) } Bound::Excluded(val) => { - inner.bound_key(table_codec, params, val.value()?, !is_upper, buffer)?; + inner + .bound_key(table_codec, params, val.value()?, !is_upper, buffer) + .await?; Ok(Bound::Excluded(buffer.as_slice())) } Bound::Unbounded => Ok(Bound::Unbounded), } } +#[maybe_async] trait IndexImpl { - fn index_lookup_into( + async fn index_lookup_into( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, @@ -1342,7 +1465,7 @@ trait IndexImpl { cover_mapping: Option<&TupleMapping>, ) -> Result<(), DatabaseError>; - fn eq_to_res<'a>( + async fn eq_to_res<'a>( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, @@ -1352,7 +1475,7 @@ trait IndexImpl { encode_max: &mut Bytes, ) -> Result, DatabaseError>; - fn bound_key( + async fn bound_key( &self, table_codec: &mut TableCodec, params: &IndexImplParams, @@ -1452,6 +1575,7 @@ struct IndexImplParams<'a, T: StorageTransaction> { reverse: bool, } +#[maybe_async] impl IndexImplParams<'_, T> { #[inline] pub(crate) fn index_meta(&self) -> &IndexMeta { @@ -1464,30 +1588,32 @@ impl IndexImplParams<'_, T> { } #[inline] - fn get_tuple_by_id_into( + async fn get_tuple_by_id_into( &self, table_codec: &mut TableCodec, tuple_id: &TupleId, tuple: &mut Tuple, ) -> Result { - table_codec.with_tuple_unchecked(self.table_id, tuple_id, None, |key, _, pool| { - let Some(bytes) = self.tx.get_borrowed(key)? else { - return Ok(false); - }; - // SAFETY: `bytes` was read from `self.tx`. - unsafe { - TableCodec::decode_tuple_into( - tuple, - self.deserializers.deserializers(), - Some(tuple_id.clone()), - bytes.as_ref(), - self.deserializers.total_len, - pool, - self.tx.mapped_scope(), - )? - }; - Ok(true) - }) + table_codec + .with_tuple_unchecked(self.table_id, tuple_id, None, async |key, _, pool| { + let Some(bytes) = self.tx.get_borrowed(key).await? else { + return Ok(false); + }; + // SAFETY: `bytes` was read from `self.tx`. + unsafe { + TableCodec::decode_tuple_into( + tuple, + self.deserializers.deserializers(), + Some(tuple_id.clone()), + bytes.as_ref(), + self.deserializers.total_len, + pool, + self.tx.mapped_scope(), + )? + }; + Ok(true) + }) + .await } } @@ -1502,8 +1628,9 @@ pub enum RangeIter<'a, T: StorageTransaction + 'a> { Reverse(T::RevIterType<'a>), } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> RangeIter<'a, T> { - fn open( + async fn open( tx: &'a T, min: Bound<&[u8]>, max: Bound<&[u8]>, @@ -1511,9 +1638,9 @@ impl<'a, T: StorageTransaction + 'a> RangeIter<'a, T> { pool: &mut BytesPool, ) -> Result { Ok(if reverse { - RangeIter::Reverse(tx.range_rev(min, max, pool)?) + RangeIter::Reverse(tx.range_rev(min, max, pool).await?) } else { - RangeIter::Forward(tx.range(min, max, pool)?) + RangeIter::Forward(tx.range(min, max, pool).await?) }) } @@ -1525,45 +1652,57 @@ impl<'a, T: StorageTransaction + 'a> RangeIter<'a, T> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> InnerIter for RangeIter<'a, T> { - fn try_next(&mut self) -> Result>, DatabaseError> { + async fn try_next(&mut self) -> Result>, DatabaseError> { match self { - RangeIter::Forward(iter) => iter.try_next(), - RangeIter::Reverse(iter) => iter.try_next(), + RangeIter::Forward(iter) => iter.try_next().await, + RangeIter::Reverse(iter) => iter.try_next().await, } } } +#[maybe_async] impl IndexImpl for IndexImplEnum { - fn index_lookup_into( + async fn index_lookup_into( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, key: &[u8], value: &[u8], - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, cover_mapping: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { match self { IndexImplEnum::PrimaryKey(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + inner + .index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + .await } IndexImplEnum::Unique(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + inner + .index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + .await } IndexImplEnum::Normal(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + inner + .index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + .await } IndexImplEnum::Composite(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + inner + .index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + .await } IndexImplEnum::Covered(inner) => { - inner.index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + inner + .index_lookup_into(table_codec, tuple, key, value, params, cover_mapping) + .await } } } - fn eq_to_res<'a>( + async fn eq_to_res<'a>( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, @@ -1574,59 +1713,80 @@ impl IndexImpl for IndexImplEnum { ) -> Result, DatabaseError> { match self { IndexImplEnum::PrimaryKey(inner) => { - inner.eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + inner + .eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + .await } IndexImplEnum::Unique(inner) => { - inner.eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + inner + .eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + .await } IndexImplEnum::Normal(inner) => { - inner.eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + inner + .eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + .await } IndexImplEnum::Composite(inner) => { - inner.eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + inner + .eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + .await } IndexImplEnum::Covered(inner) => { - inner.eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + inner + .eq_to_res(table_codec, tuple, value, params, encode_min, encode_max) + .await } } } - fn bound_key( + async fn bound_key( &self, table_codec: &mut TableCodec, - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, value: &DataValue, is_upper: bool, out: &mut Bytes, ) -> Result<(), DatabaseError> { match self { IndexImplEnum::PrimaryKey(inner) => { - inner.bound_key(table_codec, params, value, is_upper, out) + inner + .bound_key(table_codec, params, value, is_upper, out) + .await } IndexImplEnum::Unique(inner) => { - inner.bound_key(table_codec, params, value, is_upper, out) + inner + .bound_key(table_codec, params, value, is_upper, out) + .await } IndexImplEnum::Normal(inner) => { - inner.bound_key(table_codec, params, value, is_upper, out) + inner + .bound_key(table_codec, params, value, is_upper, out) + .await } IndexImplEnum::Composite(inner) => { - inner.bound_key(table_codec, params, value, is_upper, out) + inner + .bound_key(table_codec, params, value, is_upper, out) + .await } IndexImplEnum::Covered(inner) => { - inner.bound_key(table_codec, params, value, is_upper, out) + inner + .bound_key(table_codec, params, value, is_upper, out) + .await } } } } +#[maybe_async] impl IndexImpl for PrimaryKeyIndexImpl { - fn index_lookup_into( + async fn index_lookup_into( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, key: &[u8], value: &[u8], - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, _: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { let tuple_id = params @@ -1647,7 +1807,7 @@ impl IndexImpl for PrimaryKeyIndexImpl { } } - fn eq_to_res<'a>( + async fn eq_to_res<'a>( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, @@ -1656,11 +1816,11 @@ impl IndexImpl for PrimaryKeyIndexImpl { _: &mut Bytes, _: &mut Bytes, ) -> Result, DatabaseError> { - let found = - table_codec.with_tuple_unchecked(params.table_id, value, None, |key, _, pool| { + let found = table_codec + .with_tuple_unchecked(params.table_id, value, None, async |key, _, pool| { let bytes = match ¶ms.cursor { - Some(cursor) => params.tx.get_at(cursor, key)?, - None => params.tx.get_borrowed(key)?, + Some(cursor) => params.tx.get_at(cursor, key).await?, + None => params.tx.get_borrowed(key).await?, }; let Some(bytes) = bytes else { return Ok(false); @@ -1678,7 +1838,8 @@ impl IndexImpl for PrimaryKeyIndexImpl { )? }; Ok(true) - })?; + }) + .await?; Ok(if found { IndexResult::Hit } else { @@ -1686,50 +1847,57 @@ impl IndexImpl for PrimaryKeyIndexImpl { }) } - fn bound_key( + async fn bound_key( &self, table_codec: &mut TableCodec, - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, value: &DataValue, is_upper: bool, out: &mut Bytes, ) -> Result<(), DatabaseError> { - table_codec.with_tuple_unchecked(params.table_id, value, None, |key, _, _| { - encode_bound_key(out, key, is_upper); - Ok(()) - }) + table_codec + .with_tuple_unchecked(params.table_id, value, None, async |key, _, _| { + encode_bound_key(out, key, is_upper); + Ok(()) + }) + .await } } +#[maybe_async] #[inline(always)] -fn secondary_index_lookup( +async fn secondary_index_lookup( table_codec: &mut TableCodec, tuple: &mut Tuple, bytes: &[u8], - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, ) -> Result<(), DatabaseError> { let tuple_id = TableCodec::decode_index(bytes)?; - if params.get_tuple_by_id_into(table_codec, &tuple_id, tuple)? { + if params + .get_tuple_by_id_into(table_codec, &tuple_id, tuple) + .await? + { Ok(()) } else { Err(DatabaseError::TupleIdNotFound(tuple_id)) } } +#[maybe_async] impl IndexImpl for UniqueIndexImpl { - fn index_lookup_into( + async fn index_lookup_into( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, _: &[u8], value: &[u8], - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, _: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { - secondary_index_lookup(table_codec, tuple, value, params) + secondary_index_lookup(table_codec, tuple, value, params).await } - fn eq_to_res<'a>( + async fn eq_to_res<'a>( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, @@ -1739,57 +1907,65 @@ impl IndexImpl for UniqueIndexImpl { _: &mut Bytes, ) -> Result, DatabaseError> { let index = Index::new(params.index_meta().id, value.as_slice(), IndexType::Unique); - let Some(bytes) = table_codec.with_index(params.table_id, &index, None, |key, _| { - params.tx.get_borrowed(key) - })? + let Some(bytes) = table_codec + .with_index(params.table_id, &index, None, async |key, _| { + params.tx.get_borrowed(key).await + }) + .await? else { return Ok(IndexResult::Miss); }; let tuple_id = TableCodec::decode_index(bytes.as_ref())?; - if params.get_tuple_by_id_into(table_codec, &tuple_id, tuple)? { + if params + .get_tuple_by_id_into(table_codec, &tuple_id, tuple) + .await? + { Ok(IndexResult::Hit) } else { Err(DatabaseError::TupleIdNotFound(tuple_id)) } } - fn bound_key( + async fn bound_key( &self, table_codec: &mut TableCodec, - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, value: &DataValue, is_upper: bool, out: &mut Bytes, ) -> Result<(), DatabaseError> { let index = Index::new(params.index_meta().id, value.as_slice(), IndexType::Unique); - table_codec.with_index(params.table_id, &index, None, |key, _| { - if index.values.len() < params.index_meta().column_ids.len() { - // A partial UNIQUE key bounds a prefix, not one exact index entry. - encode_bound_key(out, key, is_upper); - } else { - out.clear(); - out.extend_from_slice(key); - } - Ok(()) - }) + table_codec + .with_index(params.table_id, &index, None, async |key, _| { + if index.values.len() < params.index_meta().column_ids.len() { + // A partial UNIQUE key bounds a prefix, not one exact index entry. + encode_bound_key(out, key, is_upper); + } else { + out.clear(); + out.extend_from_slice(key); + } + Ok(()) + }) + .await } } +#[maybe_async] impl IndexImpl for NormalIndexImpl { - fn index_lookup_into( + async fn index_lookup_into( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, _: &[u8], value: &[u8], - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, _: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { - secondary_index_lookup(table_codec, tuple, value, params) + secondary_index_lookup(table_codec, tuple, value, params).await } - fn eq_to_res<'a>( + async fn eq_to_res<'a>( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, @@ -1807,38 +1983,42 @@ impl IndexImpl for NormalIndexImpl { encode_min, encode_max, ) + .await } - fn bound_key( + async fn bound_key( &self, table_codec: &mut TableCodec, - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, value: &DataValue, is_upper: bool, out: &mut Bytes, ) -> Result<(), DatabaseError> { let index = Index::new(params.index_meta().id, value.as_slice(), IndexType::Normal); - table_codec.with_index(params.table_id, &index, None, |key, _| { - encode_bound_key(out, key, is_upper); - Ok(()) - }) + table_codec + .with_index(params.table_id, &index, None, async |key, _| { + encode_bound_key(out, key, is_upper); + Ok(()) + }) + .await } } +#[maybe_async] impl IndexImpl for CompositeIndexImpl { - fn index_lookup_into( + async fn index_lookup_into( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, _: &[u8], value: &[u8], - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, _: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { - secondary_index_lookup(table_codec, tuple, value, params) + secondary_index_lookup(table_codec, tuple, value, params).await } - fn eq_to_res<'a>( + async fn eq_to_res<'a>( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, @@ -1856,12 +2036,13 @@ impl IndexImpl for CompositeIndexImpl { encode_min, encode_max, ) + .await } - fn bound_key( + async fn bound_key( &self, table_codec: &mut TableCodec, - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, value: &DataValue, is_upper: bool, out: &mut Bytes, @@ -1871,21 +2052,24 @@ impl IndexImpl for CompositeIndexImpl { value.as_slice(), IndexType::Composite, ); - table_codec.with_index(params.table_id, &index, None, |key, _| { - encode_bound_key(out, key, is_upper); - Ok(()) - }) + table_codec + .with_index(params.table_id, &index, None, async |key, _| { + encode_bound_key(out, key, is_upper); + Ok(()) + }) + .await } } +#[maybe_async] impl IndexImpl for CoveredIndexImpl { - fn index_lookup_into( + async fn index_lookup_into( &self, _: &mut TableCodec, tuple: &mut Tuple, key: &[u8], value: &[u8], - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, cover_mapping: Option<&TupleMapping>, ) -> Result<(), DatabaseError> { let ty = params.value_ty(); @@ -1909,7 +2093,7 @@ impl IndexImpl for CoveredIndexImpl { } } - fn eq_to_res<'a>( + async fn eq_to_res<'a>( &self, table_codec: &mut TableCodec, tuple: &mut Tuple, @@ -1927,12 +2111,13 @@ impl IndexImpl for CoveredIndexImpl { encode_min, encode_max, ) + .await } - fn bound_key( + async fn bound_key( &self, table_codec: &mut TableCodec, - params: &IndexImplParams, + params: &IndexImplParams<'_, T>, value: &DataValue, is_upper: bool, out: &mut Bytes, @@ -1942,15 +2127,18 @@ impl IndexImpl for CoveredIndexImpl { value.as_slice(), params.index_meta().ty, ); - table_codec.with_index(params.table_id, &index, None, |key, _| { - encode_bound_key(out, key, is_upper); - Ok(()) - }) + table_codec + .with_index(params.table_id, &index, None, async |key, _| { + encode_bound_key(out, key, is_upper); + Ok(()) + }) + .await } } +#[maybe_async] #[inline(always)] -fn eq_to_res_scope<'a, T: StorageTransaction + 'a>( +async fn eq_to_res_scope<'a, T: StorageTransaction + 'a>( table_codec: &mut TableCodec, tuple: &mut Tuple, index_impl: &impl IndexImpl, @@ -1960,8 +2148,12 @@ fn eq_to_res_scope<'a, T: StorageTransaction + 'a>( encode_max: &mut Bytes, ) -> Result, DatabaseError> { let _ = tuple; - index_impl.bound_key(table_codec, params, value, false, encode_min)?; - index_impl.bound_key(table_codec, params, value, true, encode_max)?; + index_impl + .bound_key(table_codec, params, value, false, encode_min) + .await?; + index_impl + .bound_key(table_codec, params, value, true, encode_max) + .await?; let iter = RangeIter::open( params.tx, @@ -1969,7 +2161,8 @@ fn eq_to_res_scope<'a, T: StorageTransaction + 'a>( Bound::Included(encode_max.as_slice()), params.reverse, table_codec.bytes_pool(), - )?; + ) + .await?; Ok(IndexResult::Scope(iter)) } @@ -2088,20 +2281,24 @@ pub struct TupleIter<'a, T: StorageTransaction + 'a> { mapped: Option<&'a MappedScope>, } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> TupleIter<'a, T> { pub(crate) fn recycle(self, pool: &mut BytesPool) { self.iter.recycle(pool); } - pub(crate) fn reset( + pub(crate) async fn reset( &mut self, tx: &'a T, table_codec: &mut TableCodec, bounds: Bounds, ) -> Result<(), DatabaseError> { - let iter = table_codec.with_tuple_bound(self.table_id, |min, max, pool| { - tx.range(Bound::Included(min), Bound::Included(max), pool) - })?; + let iter = table_codec + .with_tuple_bound(self.table_id, async |min, max, pool| { + tx.range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; mem::replace(&mut self.iter, iter).recycle(table_codec.bytes_pool()); self.mapped = tx.mapped_scope(); self.bounds = IterBounds::new(bounds.0.unwrap_or(0), bounds.1); @@ -2109,13 +2306,14 @@ impl<'a, T: StorageTransaction + 'a> TupleIter<'a, T> { } } +#[maybe_async] impl<'a, T: StorageTransaction + 'a> Iter for TupleIter<'a, T> { - fn next_tuple_into( + async fn next_tuple_into( &mut self, table_codec: &mut TableCodec, tuple: &mut Tuple, ) -> Result { - while let Some((key, value)) = self.iter.try_next()? { + while let Some((key, value)) = self.iter.try_next().await? { if table_codec.is_own_write(value) || self.bounds.consume_offset() { continue; } @@ -2199,6 +2397,7 @@ pub enum IndexIterState<'a, T: StorageTransaction + 'a> { Over, } +#[maybe_async] impl<'a, T: StorageTransaction> IndexIter<'a, T> { pub(crate) fn recycle(self, pool: &mut BytesPool) { if let IndexIterState::Range(iter) = self.state { @@ -2224,7 +2423,7 @@ impl<'a, T: StorageTransaction> IndexIter<'a, T> { Some(range) } - pub(crate) fn next_tuple_into( + pub(crate) async fn next_tuple_into( &mut self, view: IndexScanView<'_>, bound_buffers: &mut [Bytes; 2], @@ -2256,7 +2455,8 @@ impl<'a, T: StorageTransaction> IndexIter<'a, T> { encode_min, &self.params, &self.inner, - )?; + ) + .await?; let encode_max = encode_bound( table_codec, max, @@ -2264,12 +2464,15 @@ impl<'a, T: StorageTransaction> IndexIter<'a, T> { encode_max, &self.params, &self.inner, - )?; + ) + .await?; let tx = self.params.tx; let reverse = self.params.reverse; let open_iter = - move |bound_min: &[u8], bound_max: &[u8], pool: &mut BytesPool| { + async move |bound_min: &[u8], + bound_max: &[u8], + pool: &mut BytesPool| { RangeIter::open( tx, fill_default_bound(encode_min, bound_min), @@ -2277,11 +2480,14 @@ impl<'a, T: StorageTransaction> IndexIter<'a, T> { reverse, pool, ) + .await }; let iter = if matches!(index_meta.ty, IndexType::PrimaryKey { .. }) { - table_codec.with_tuple_bound(table_id, open_iter)? + table_codec.with_tuple_bound(table_id, open_iter).await? } else { - table_codec.with_index_bound(table_id, index_meta.id, open_iter)? + table_codec + .with_index_bound(table_id, index_meta.id, open_iter) + .await? }; self.state = IndexIterState::Range(iter); } @@ -2292,14 +2498,18 @@ impl<'a, T: StorageTransaction> IndexIter<'a, T> { { self.params.cursor = Some(self.params.tx.point_cursor()?); } - match self.inner.eq_to_res( - table_codec, - tuple, - val.value()?, - &self.params, - encode_min, - encode_max, - )? { + match self + .inner + .eq_to_res( + table_codec, + tuple, + val.value()?, + &self.params, + encode_min, + encode_max, + ) + .await? + { IndexResult::Hit => { if self.bounds.consume_offset() { continue; @@ -2317,19 +2527,21 @@ impl<'a, T: StorageTransaction> IndexIter<'a, T> { } } IndexIterState::Range(iter) => { - while let Some((key, value)) = iter.try_next()? { + while let Some((key, value)) = iter.try_next().await? { if table_codec.is_own_write(value) || self.bounds.consume_offset() { continue; } self.bounds.consume_limit(); - self.inner.index_lookup_into( - table_codec, - tuple, - key, - value, - &self.params, - view.cover_mapping, - )?; + self.inner + .index_lookup_into( + table_codec, + tuple, + key, + value, + &self.params, + view.cover_mapping, + ) + .await?; return Ok(true); } @@ -2345,8 +2557,9 @@ impl<'a, T: StorageTransaction> IndexIter<'a, T> { } } -pub trait InnerIter { - fn try_next(&mut self) -> Result>, DatabaseError>; +#[maybe_async] +pub trait InnerIter: MaybeSend { + async fn try_next(&mut self) -> Result>, DatabaseError>; #[doc(hidden)] fn recycle(self, pool: &mut BytesPool) @@ -2357,8 +2570,9 @@ pub trait InnerIter { } } -pub trait Iter { - fn next_tuple_into( +#[maybe_async] +pub trait Iter: MaybeSend { + async fn next_tuple_into( &mut self, table_codec: &mut TableCodec, tuple: &mut Tuple, @@ -2369,13 +2583,14 @@ pub struct TableIter<'a, T: StorageTransaction + 'a> { pub(crate) iter: T::IterType<'a>, } +#[maybe_async] impl TableIter<'_, T> { - pub fn try_next( + pub async fn try_next( &mut self, arena: &mut (impl MetaArena + ?Sized), ) -> Result, DatabaseError> { // Views share the root namespace. - while let Some((_, value)) = self.iter.try_next()? { + while let Some((_, value)) = self.iter.try_next().await? { let meta = TableCodec::decode_root_table::(value, arena)?; if !meta.is_view { return Ok(Some(meta)); @@ -2389,12 +2604,13 @@ pub struct ViewIter<'a, T: StorageTransaction + 'a> { pub(crate) iter: T::IterType<'a>, } +#[maybe_async] impl ViewIter<'_, T> { - pub fn try_next( + pub async fn try_next( &mut self, arena: &mut (impl MetaArena + ?Sized), ) -> Result, DatabaseError> { - let Some((_, value)) = self.iter.try_next()? else { + let Some((_, value)) = self.iter.try_next().await? else { return Ok(None); }; @@ -2403,13 +2619,14 @@ impl ViewIter<'_, T> { } // GRCOV_EXCL_START +#[maybe_async] #[cfg(test)] -pub(crate) fn check_range_rev_matches_range( +pub(crate) async fn check_range_rev_matches_range( tx: &mut T, ) -> Result<(), DatabaseError> { - fn keys(mut iter: I) -> Result>, DatabaseError> { + async fn keys(mut iter: I) -> Result>, DatabaseError> { let mut keys = Vec::new(); - while let Some((key, _)) = iter.try_next()? { + while let Some((key, _)) = iter.try_next().await? { keys.push(key.to_vec()); } Ok(keys) @@ -2467,37 +2684,45 @@ pub(crate) fn check_range_rev_matches_range( .cloned() .collect(); assert_eq!( - keys(tx.range(*min, *max, &mut pool)?)?, + keys(tx.range(*min, *max, &mut pool).await?).await?, expected, "min={min:?} max={max:?}" ); expected.reverse(); - let actual = keys(tx.range_rev(*min, *max, &mut pool)?)?; + let actual = keys(tx.range_rev(*min, *max, &mut pool).await?).await?; assert_eq!(actual, expected, "min={min:?} max={max:?}"); } } Ok(()) } +#[maybe_async] #[cfg(test)] -pub(crate) fn check_explicit_transaction_does_not_rescan_own_writes( +pub(crate) async fn check_explicit_transaction_does_not_rescan_own_writes( mut db: crate::db::Database, ) -> Result<(), DatabaseError> { - db.ddl("create table t (id int primary key, v int not null)")?; + db.ddl("create table t (id int primary key, v int not null)") + .await?; let mut tx = db.new_transaction(false)?; - tx.run("insert into t values (1,1), (2,2), (3,3), (4,4), (5,5)")? - .done()?; - tx.run("update t set id = id + 1000 where id > 3 and id < 3000")? - .done()?; - tx.run("insert into t select id + 1000, v from t where id < 3000")? - .done()?; - let mut iter = tx.run("select id from t order by id")?; + tx.run("insert into t values (1,1), (2,2), (3,3), (4,4), (5,5)") + .await? + .done() + .await?; + tx.run("update t set id = id + 1000 where id > 3 and id < 3000") + .await? + .done() + .await?; + tx.run("insert into t select id + 1000, v from t where id < 3000") + .await? + .done() + .await?; + let mut iter = tx.run("select id from t order by id").await?; let mut ids = Vec::new(); - while let Some(id) = iter.next_tuple(|_, tuple| tuple[0].clone())? { + while let Some(id) = iter.next_tuple(|_, tuple| tuple[0].clone()).await? { ids.push(id); } - iter.done()?; - tx.commit()?; + iter.done().await?; + tx.commit().await?; let expected = [1, 2, 3, 1001, 1002, 1003, 1004, 1005, 2004, 2005] .map(DataValue::Int32) .to_vec(); @@ -2505,8 +2730,53 @@ pub(crate) fn check_explicit_transaction_does_not_rescan_own_writes( Ok(()) } +#[cfg(all(test, not(feature = "async")))] +pub(crate) type TestStorage = crate::storage::memory::MemoryStorage; + +#[cfg(all(test, not(feature = "async")))] +pub(crate) type TestTransaction = crate::storage::memory::MemoryTransaction; + +#[cfg(all(test, feature = "async"))] +pub(crate) type TestStorage = crate::storage::slatedb::SlateDbStorage; + +#[cfg(all(test, feature = "async"))] +pub(crate) type TestTransaction = crate::storage::slatedb::SlateDbTransaction; + +#[cfg(all(test, not(feature = "async")))] +pub(crate) fn new_test_storage() -> TestStorage { + TestStorage::new() +} + +#[cfg(all(test, feature = "async"))] +pub(crate) async fn new_test_storage() -> TestStorage { + use ::slatedb::object_store::memory::InMemory; + use std::sync::Arc; + + TestStorage::open( + "kitesql-test", + Arc::new(InMemory::new()), + Default::default(), + ) + .await + .expect("failed to open SlateDB on an in-memory object store") + .await_durable(false) +} + +// The async path's stand-in for `DataBaseBuilder::build_in_memory` in unit tests. +#[cfg(all(test, feature = "async"))] +impl crate::db::DataBaseBuilder { + pub(crate) async fn build_in_memory( + self, + ) -> Result, DatabaseError> { + self.build_with_storage(new_test_storage().await) + } +} + +#[maybe_async] #[cfg(test)] -pub(crate) fn check_remove_range(tx: &mut T) -> Result<(), DatabaseError> { +pub(crate) async fn check_remove_range( + tx: &mut T, +) -> Result<(), DatabaseError> { let data: Vec<[u8; 4]> = (0u32..3000).map(u32::to_be_bytes).collect(); let [k0, k1, k10, k2500, k2999, k5000] = [0u32, 1, 10, 2500, 2999, 5000].map(u32::to_be_bytes); type Case<'a> = (Bound<&'a [u8]>, Bound<&'a [u8]>); @@ -2524,11 +2794,13 @@ pub(crate) fn check_remove_range(tx: &mut T) -> Result<() for key in &data { tx.set(key, &[7; 64])?; } - tx.remove_range(min, max, &mut pool)?; + tx.remove_range(min, max, &mut pool).await?; - let mut iter = tx.range(Bound::Unbounded, Bound::Unbounded, &mut pool)?; + let mut iter = tx + .range(Bound::Unbounded, Bound::Unbounded, &mut pool) + .await?; let mut remaining = Vec::new(); - while let Some((key, _)) = iter.try_next()? { + while let Some((key, _)) = iter.try_next().await? { remaining.push(key.to_vec()); } let expected: Vec> = data @@ -2583,16 +2855,18 @@ pub(crate) fn deserializers_for_test( } /// Reads one tuple with `next`, which is given a fresh codec and tuple. +#[maybe_async] #[cfg(test)] -pub(crate) fn next_tuple_for_test( - mut next: impl FnMut(&mut TableCodec, &mut Tuple) -> Result, +pub(crate) async fn next_tuple_for_test( + mut next: impl AsyncFnMut(&mut TableCodec, &mut Tuple) -> Result, ) -> Result, DatabaseError> { let mut tuple = Tuple::default(); let mut table_codec = TableCodec::default(); - Ok(next(&mut table_codec, &mut tuple)?.then_some(tuple)) + Ok(next(&mut table_codec, &mut tuple).await?.then_some(tuple)) } #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use crate::binder::test::build_t1_table; use crate::catalog::view::View; @@ -2603,9 +2877,9 @@ mod test { use crate::errors::DatabaseError; use crate::expression::range_detacher::Range; use crate::planner::{PlanArena, TableArenaCell}; - use crate::storage::rocksdb::{RocksStorage, RocksTransaction}; use crate::storage::table_codec::TableCodec; use crate::storage::NameMapping; + use crate::storage::TestTransaction; use crate::storage::{ project_deserializers_for_test, IndexIter, IndexScanView, InnerIter, Iter, Storage, StorageTransaction, TableCache, TupleDeserializers, @@ -2616,7 +2890,6 @@ mod test { use crate::types::LogicalType; use std::collections::Bound; use std::sync::Arc; - use tempfile::TempDir; fn full_columns(names: &NameMapping, table_cache: &TableCache) -> Vec { names @@ -2657,9 +2930,8 @@ mod test { } #[test] - fn test_table_create_drop() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_table_create_drop() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let mut transaction = storage.transaction()?; let mut names = crate::storage::NameMapping::default(); let mut table_cache = crate::storage::TableCache::default(); @@ -2671,10 +2943,11 @@ mod test { &mut table_cache, &mut transaction, &mut plan_arena, - )?; + ) + .await?; let plan_arena = PlanArena::new(&table_arena); - let fn_assert = |transaction: &mut RocksTransaction, + let fn_assert = |transaction: &mut TestTransaction, names: &NameMapping, table_cache: &TableCache, plan_arena: &PlanArena| @@ -2758,7 +3031,8 @@ mod test { let mut table_codec = TableCodec::default(); let table_name = "t1".to_string(); let table = transaction - .load_table(&mut table_codec, table_arena.borrow_mut(), &table_name)? + .load_table(&mut table_codec, table_arena.borrow_mut(), &table_name) + .await? .ok_or(DatabaseError::TableNotFound)?; reloaded_names.insert_table(&mut reloaded_table_cache, table); let plan_arena = PlanArena::new(&table_arena); @@ -2773,10 +3047,9 @@ mod test { } #[test] - fn test_tuple_append_delete() -> Result<(), DatabaseError> { + async fn test_tuple_append_delete() -> Result<(), DatabaseError> { let mut table_codec = TableCodec::default(); - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + let storage = crate::storage::new_test_storage().await; let mut transaction = storage.transaction()?; let mut names = crate::storage::NameMapping::default(); let mut table_cache = crate::storage::TableCache::default(); @@ -2788,22 +3061,25 @@ mod test { &mut table_cache, &mut transaction, &mut plan_arena, - )?; + ) + .await?; let plan_arena = PlanArena::new(&table_arena); let tuples = build_tuples(); for tuple in build_tuples() { - transaction.append_tuple( - &mut table_codec, - 0, - &tuple.into(), - &[ - LogicalType::Integer.serializable(), - LogicalType::Boolean.serializable(), - LogicalType::Integer.serializable(), - ], - false, - )?; + transaction + .append_tuple( + &mut table_codec, + 0, + &tuple.into(), + &[ + LogicalType::Integer.serializable(), + LogicalType::Boolean.serializable(), + LogicalType::Integer.serializable(), + ], + false, + ) + .await?; } { let deserializers = project_deserializers_for_test( @@ -2814,40 +3090,60 @@ mod test { &plan_arena, true, ); - let mut tuple_iter = transaction.read( - &mut table_codec, - &deserializers, - names.table(&table_cache, "t1").unwrap().id(), - (None, None), - )?; + let mut tuple_iter = transaction + .read( + &mut table_codec, + &deserializers, + names.table(&table_cache, "t1").unwrap().id(), + (None, None), + ) + .await?; assert_eq!( - super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), + super::next_tuple_for_test(async |codec, tuple| { + tuple_iter.next_tuple_into(codec, tuple).await + }) + .await? + .unwrap(), tuples[0] ); assert_eq!( - super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), + super::next_tuple_for_test(async |codec, tuple| { + tuple_iter.next_tuple_into(codec, tuple).await + }) + .await? + .unwrap(), tuples[1] ); assert_eq!( - super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), + super::next_tuple_for_test(async |codec, tuple| { + tuple_iter.next_tuple_into(codec, tuple).await + }) + .await? + .unwrap(), tuples[2] ); - let mut iter = table_codec.with_tuple_bound(0, |min, max, pool| { - transaction.range(Bound::Included(min), Bound::Included(max), pool) - })?; + let mut iter = table_codec + .with_tuple_bound(0, async |min, max, pool| { + transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - assert!(iter.try_next()?.is_none()); + assert!(iter.try_next().await?.is_none()); } - transaction.remove_tuple(&mut table_codec, 0, &tuples[1].values[0])?; + transaction + .remove_tuple(&mut table_codec, 0, &tuples[1].values[0]) + .await?; { let deserializers = project_deserializers_for_test( &names, @@ -2857,41 +3153,54 @@ mod test { &plan_arena, true, ); - let mut tuple_iter = transaction.read( - &mut table_codec, - &deserializers, - names.table(&table_cache, "t1").unwrap().id(), - (None, None), - )?; + let mut tuple_iter = transaction + .read( + &mut table_codec, + &deserializers, + names.table(&table_cache, "t1").unwrap().id(), + (None, None), + ) + .await?; assert_eq!( - super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), + super::next_tuple_for_test(async |codec, tuple| { + tuple_iter.next_tuple_into(codec, tuple).await + }) + .await? + .unwrap(), tuples[0] ); assert_eq!( - super::next_tuple_for_test(|codec, tuple| tuple_iter.next_tuple_into(codec, tuple))?.unwrap(), + super::next_tuple_for_test(async |codec, tuple| { + tuple_iter.next_tuple_into(codec, tuple).await + }) + .await? + .unwrap(), tuples[2] ); - let mut iter = table_codec.with_tuple_bound(0, |min, max, pool| { - transaction.range(Bound::Included(min), Bound::Included(max), pool) - })?; + let mut iter = table_codec + .with_tuple_bound(0, async |min, max, pool| { + transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - assert!(iter.try_next()?.is_none()); + assert!(iter.try_next().await?.is_none()); } Ok(()) } #[test] - fn test_add_index_meta() -> Result<(), DatabaseError> { + async fn test_add_index_meta() -> Result<(), DatabaseError> { let mut table_codec = TableCodec::default(); - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + let storage = crate::storage::new_test_storage().await; let mut transaction = storage.transaction()?; let mut names = crate::storage::NameMapping::default(); let mut table_cache = crate::storage::TableCache::default(); @@ -2903,7 +3212,8 @@ mod test { &mut table_cache, &mut transaction, &mut plan_arena, - )?; + ) + .await?; let mut plan_arena = PlanArena::new(&table_arena); let (c2_column_id, c3_column_id) = { let t1_table = transaction.table(&names, &table_cache, "t1")?.unwrap(); @@ -2914,29 +3224,33 @@ mod test { ) }; - let (table, _) = transaction.add_index_meta( - &mut table_codec, - &mut plan_arena, - "t1", - "i1".to_string(), - vec![c3_column_id], - IndexType::Normal, - )?; + let (table, _) = transaction + .add_index_meta( + &mut table_codec, + &mut plan_arena, + "t1", + "i1".to_string(), + vec![c3_column_id], + IndexType::Normal, + ) + .await?; let table = table.transplant_to_table_arena(&plan_arena)?; names.insert_table(&mut table_cache, table); let mut plan_arena = PlanArena::new(&table_arena); - let (table, _) = transaction.add_index_meta( - &mut table_codec, - &mut plan_arena, - "t1", - "i2".to_string(), - vec![c3_column_id, c2_column_id], - IndexType::Composite, - )?; + let (table, _) = transaction + .add_index_meta( + &mut table_codec, + &mut plan_arena, + "t1", + "i2".to_string(), + vec![c3_column_id, c2_column_id], + IndexType::Composite, + ) + .await?; let table = table.transplant_to_table_arena(&plan_arena)?; names.insert_table(&mut table_cache, table); - let fn_assert = |transaction: &mut RocksTransaction, + let fn_assert = |transaction: &mut TestTransaction, names: &NameMapping, table_cache: &TableCache| -> Result<(), DatabaseError> { @@ -2966,30 +3280,39 @@ mod test { let mut reloaded_table_cache = crate::storage::TableCache::default(); let table_name = "t1".to_string(); let table = transaction - .load_table(&mut table_codec, table_arena.borrow_mut(), &table_name)? + .load_table(&mut table_codec, table_arena.borrow_mut(), &table_name) + .await? .ok_or(DatabaseError::TableNotFound)?; reloaded_names.insert_table(&mut reloaded_table_cache, table); let mut plan_arena = PlanArena::new(&table_arena); fn_assert(&mut transaction, &reloaded_names, &reloaded_table_cache)?; { - let mut iter = table_codec.with_index_meta_bound(0, |min, max, pool| { - transaction.range(Bound::Included(min), Bound::Included(max), pool) - })?; + let mut iter = table_codec + .with_index_meta_bound(0, async |min, max, pool| { + transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - assert!(iter.try_next()?.is_none()); + assert!(iter.try_next().await?.is_none()); } - match transaction.drop_index(&mut table_codec, &mut plan_arena, "t1", "pk_index", false) { + match transaction + .drop_index(&mut table_codec, &mut plan_arena, "t1", "pk_index", false) + .await + { Err(DatabaseError::InvalidIndex) => (), _ => unreachable!(), } - if let Some((table, _)) = - transaction.drop_index(&mut table_codec, &mut plan_arena, "t1", "i1", false)? + if let Some((table, _)) = transaction + .drop_index(&mut table_codec, &mut plan_arena, "t1", "i1", false) + .await? { let table = table.transplant_to_table_arena(&plan_arena)?; names.insert_table(&mut table_cache, table); @@ -3004,29 +3327,33 @@ mod test { assert_eq!(i2_meta.name, "i2".to_string()); assert_eq!(i2_meta.ty, IndexType::Composite); - let mut iter = table_codec.with_index_meta_bound(0, |min, max, pool| { - transaction.range(Bound::Included(min), Bound::Included(max), pool) - })?; + let mut iter = table_codec + .with_index_meta_bound(0, async |min, max, pool| { + transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - assert!(iter.try_next()?.is_none()); + assert!(iter.try_next().await?.is_none()); } Ok(()) } #[test] - fn test_index_insert_delete() -> Result<(), DatabaseError> { + async fn test_index_insert_delete() -> Result<(), DatabaseError> { fn build_index_iter<'a>( - transaction: &'a RocksTransaction<'a>, + transaction: &'a TestTransaction, names: &NameMapping, table_cache: &'a TableCache, plan_arena: &'a PlanArena<'a>, deserializers: &'a TupleDeserializers, - ) -> Result>, DatabaseError> { + ) -> Result, DatabaseError> { let table_name: crate::catalog::TableName = "t1".to_string(); let index_meta = names .table(table_cache, &table_name) @@ -3054,8 +3381,7 @@ mod test { }; let mut table_codec = TableCodec::default(); - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + let storage = crate::storage::new_test_storage().await; let mut transaction = storage.transaction()?; let mut names = crate::storage::NameMapping::default(); let mut table_cache = crate::storage::TableCache::default(); @@ -3067,19 +3393,22 @@ mod test { &mut table_cache, &mut transaction, &mut plan_arena, - )?; + ) + .await?; let mut plan_arena = PlanArena::new(&table_arena); let t1_table = transaction.table(&names, &table_cache, "t1")?.unwrap(); let c3_column_id = *t1_table.get_column_id_by_name("c3").unwrap(); - let (table, _) = transaction.add_index_meta( - &mut table_codec, - &mut plan_arena, - "t1", - "i1".to_string(), - vec![c3_column_id], - IndexType::Normal, - )?; + let (table, _) = transaction + .add_index_meta( + &mut table_codec, + &mut plan_arena, + "t1", + "i1".to_string(), + vec![c3_column_id], + IndexType::Normal, + ) + .await?; let table = table.transplant_to_table_arena(&plan_arena)?; names.insert_table(&mut table_cache, table); let plan_arena = PlanArena::new(&table_arena); @@ -3112,20 +3441,24 @@ mod test { ), ]; for (tuple_id, index) in indexes.iter().cloned() { - transaction.add_index(&mut table_codec, 0, index, &tuple_id)?; + transaction + .add_index(&mut table_codec, 0, index, &tuple_id) + .await?; } for tuple in build_tuples() { - transaction.append_tuple( - &mut table_codec, - 0, - &tuple.into(), - &[ - LogicalType::Integer.serializable(), - LogicalType::Boolean.serializable(), - LogicalType::Integer.serializable(), - ], - false, - )?; + transaction + .append_tuple( + &mut table_codec, + 0, + &tuple.into(), + &[ + LogicalType::Integer.serializable(), + LogicalType::Boolean.serializable(), + LogicalType::Integer.serializable(), + ], + false, + ) + .await?; } { let deserializers = project_deserializers_for_test( @@ -3145,64 +3478,79 @@ mod test { )?; assert_eq!( - super::next_tuple_for_test(|codec, tuple| { - index_iter.next_tuple_into( - IndexScanView { - range: &full, - cover_mapping: None, - }, - &mut Default::default(), - codec, - tuple, - ) - })? + super::next_tuple_for_test(async |codec, tuple| { + index_iter + .next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + .await + }) + .await? .unwrap(), tuples[0] ); assert_eq!( - super::next_tuple_for_test(|codec, tuple| { - index_iter.next_tuple_into( - IndexScanView { - range: &full, - cover_mapping: None, - }, - &mut Default::default(), - codec, - tuple, - ) - })? + super::next_tuple_for_test(async |codec, tuple| { + index_iter + .next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + .await + }) + .await? .unwrap(), tuples[2] ); assert_eq!( - super::next_tuple_for_test(|codec, tuple| { - index_iter.next_tuple_into( - IndexScanView { - range: &full, - cover_mapping: None, - }, - &mut Default::default(), - codec, - tuple, - ) - })? + super::next_tuple_for_test(async |codec, tuple| { + index_iter + .next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + .await + }) + .await? .unwrap(), tuples[1] ); - let mut iter = table_codec.with_index_bound(0, 1, |min, max, pool| { - transaction.range(Bound::Included(min), Bound::Included(max), pool) - })?; + let mut iter = table_codec + .with_index_bound(0, 1, async |min, max, pool| { + transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - assert!(iter.try_next()?.is_none()); + assert!(iter.try_next().await?.is_none()); } - transaction.del_index(&mut table_codec, 0, &indexes[0].1, &indexes[0].0)?; + transaction + .del_index(&mut table_codec, 0, &indexes[0].1, &indexes[0].0) + .await?; let deserializers = project_deserializers_for_test( &names, @@ -3222,53 +3570,62 @@ mod test { )?; assert_eq!( - super::next_tuple_for_test(|codec, tuple| { - index_iter.next_tuple_into( - IndexScanView { - range: &full, - cover_mapping: None, - }, - &mut Default::default(), - codec, - tuple, - ) - })? + super::next_tuple_for_test(async |codec, tuple| { + index_iter + .next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + .await + }) + .await? .unwrap(), tuples[2] ); assert_eq!( - super::next_tuple_for_test(|codec, tuple| { - index_iter.next_tuple_into( - IndexScanView { - range: &full, - cover_mapping: None, - }, - &mut Default::default(), - codec, - tuple, - ) - })? + super::next_tuple_for_test(async |codec, tuple| { + index_iter + .next_tuple_into( + IndexScanView { + range: &full, + cover_mapping: None, + }, + &mut Default::default(), + codec, + tuple, + ) + .await + }) + .await? .unwrap(), tuples[1] ); - let mut iter = table_codec.with_index_bound(0, 1, |min, max, pool| { - transaction.range(Bound::Included(min), Bound::Included(max), pool) - })?; + let mut iter = table_codec + .with_index_bound(0, 1, async |min, max, pool| { + transaction + .range(Bound::Included(min), Bound::Included(max), pool) + .await + }) + .await?; - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - let (_, value) = iter.try_next()?.unwrap(); + let (_, value) = iter.try_next().await?.unwrap(); dbg!(value); - assert!(iter.try_next()?.is_none()); + assert!(iter.try_next().await?.is_none()); Ok(()) } #[test] - fn test_column_add_drop() -> Result<(), DatabaseError> { - let temp_dir = TempDir::new().expect("unable to create temporary working directory"); - let storage = RocksStorage::new(temp_dir.path())?; + async fn test_column_add_drop() -> Result<(), DatabaseError> { + let storage = crate::storage::new_test_storage().await; let mut transaction = storage.transaction()?; let mut names = crate::storage::NameMapping::default(); let mut table_cache = crate::storage::TableCache::default(); @@ -3281,7 +3638,8 @@ mod test { &mut table_cache, &mut transaction, &mut plan_arena, - )?; + ) + .await?; let mut plan_arena = PlanArena::new(&table_arena); let table_name: TableName = "t1".to_string(); @@ -3290,13 +3648,15 @@ mod test { true, ColumnDesc::new(LogicalType::Integer, None, false, None)?, ); - let (table, new_column_id) = transaction.add_column( - &mut table_codec, - &mut plan_arena, - &table_name, - &new_column, - false, - )?; + let (table, new_column_id) = transaction + .add_column( + &mut table_codec, + &mut plan_arena, + &table_name, + &new_column, + false, + ) + .await?; let table = table.transplant_to_table_arena(&plan_arena)?; names.insert_table(&mut table_cache, table); let mut plan_arena = PlanArena::new(&table_arena); @@ -3309,6 +3669,7 @@ mod test { &new_column, false, ) + .await .is_err()); assert_eq!( new_column_id, @@ -3319,7 +3680,8 @@ mod test { &table_name, &new_column, true, - )? + ) + .await? .1 ); } @@ -3341,8 +3703,9 @@ mod test { let column = table.get_column_by_name("c4").unwrap(); assert_eq!(table_arena.borrow().column(column), &new_column); } - let table = - transaction.drop_column(&mut table_codec, &mut plan_arena, &table_name, "c4")?; + let table = transaction + .drop_column(&mut table_codec, &mut plan_arena, &table_name, "c4") + .await?; let table = table.transplant_to_table_arena(&plan_arena)?; names.insert_table(&mut table_cache, table); { @@ -3357,8 +3720,8 @@ mod test { } #[test] - fn test_view_create_drop() -> Result<(), DatabaseError> { - let table_state = build_t1_table()?; + async fn test_view_create_drop() -> Result<(), DatabaseError> { + let table_state = build_t1_table().await?; let scala_functions = Default::default(); let table_functions = Default::default(); @@ -3378,7 +3741,9 @@ mod test { let mut names = NameMapping::default(); let mut view_cache = crate::storage::ViewCache::default(); let mut table_codec = TableCodec::default(); - let view_id = transaction.create_view(&mut table_codec, &mut plan_arena, &view, true)?; + let view_id = transaction + .create_view(&mut table_codec, &mut plan_arena, &view, true) + .await?; names.insert_view(&mut view_cache, view_id, view.clone()); assert_eq!( @@ -3408,10 +3773,16 @@ mod test { .unwrap() ); - if transaction.drop_view(&mut table_codec, &mut plan_arena, view_name.clone(), false)? { + if transaction + .drop_view(&mut table_codec, &mut plan_arena, view_name.clone(), false) + .await? + { names.remove(&view_name); } - if transaction.drop_view(&mut table_codec, &mut plan_arena, view_name.clone(), true)? { + if transaction + .drop_view(&mut table_codec, &mut plan_arena, view_name.clone(), true) + .await? + { names.remove(&view_name); } assert!(transaction diff --git a/src/storage/slatedb.rs b/src/storage/slatedb.rs new file mode 100644 index 00000000..d4dadef8 --- /dev/null +++ b/src/storage/slatedb.rs @@ -0,0 +1,643 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! A storage over [SlateDB](https://slatedb.io), an LSM tree kept in object storage. It runs on +//! the async path only: SlateDB drives its I/O and background work on tokio, so open the +//! database and use it inside a tokio runtime. +//! +//! With the `slatedb-s3` feature, [`SlateDbStorage::open_s3`] opens it on Amazon S3 or an +//! S3-compatible store described by an [`S3Config`]. +//! +//! Scans read their SSTs as [`SlateDbScanOptions`] set, and [`SlateDbCacheOptions`] sizes the +//! block cache [`SlateDbStorage::open`] builds and can add a local disk tier to it. For other +//! SlateDB settings, build the [`Db`] with [`Db::builder`] and wrap it with +//! [`SlateDbStorage::new`]. +//! +//! Each KiteSQL transaction is a SlateDB snapshot-isolation transaction. Reads see the snapshot +//! taken when it began plus its own writes; a commit that conflicts with a write committed since +//! fails, and the transaction must be retried. + +use crate::errors::DatabaseError; +use crate::storage::{ + EmptyStorageMetrics, InnerIter, KeyValueRef, Storage, StorageTransaction, + TransactionIsolationLevel, +}; +use crate::types::serialize::BytesPool; +use slatedb::bytes::Bytes; +use slatedb::config::ScanOptions; +use slatedb::db_cache::foyer::{FoyerCache, FoyerCacheOptions}; +use slatedb::db_cache::foyer_hybrid::FoyerHybridCache; +use slatedb::db_cache::{ + CachedEntry, DbCache, SplitCache, DEFAULT_BLOCK_CACHE_CAPACITY, DEFAULT_META_CACHE_CAPACITY, +}; +use slatedb::object_store::path::Path; +use slatedb::object_store::ObjectStore; +use slatedb::{ + BlockCachePolicy, CacheTarget, Db, DbIterator, DbTransaction, IsolationLevel, IterationOrder, + KeyValue, +}; +use std::collections::Bound; +use std::path::PathBuf; +use std::sync::Arc; + +#[derive(Clone)] +pub struct SlateDbStorage { + db: Arc, + await_durable: bool, + scan_options: SlateDbScanOptions, + cache: Option>, +} + +#[derive(Clone, Debug, Default)] +pub struct SlateDbCacheOptions { + /// The memory for data blocks; 512MiB when unset. + pub memory_bytes: Option, + /// A local directory where data blocks go when memory evicts them, so that reads of them skip + /// object storage. + pub disk: Option, + /// Put the data blocks compactions write into the cache. Without it, compacting rows drops + /// them from the cache, and their next reads go to object storage. + pub cache_compaction_output: bool, +} + +#[derive(Clone, Debug)] +pub struct SlateDbDiskCache { + /// The directory of the cache files. + pub dir: PathBuf, + /// The bytes the cache files may take. The files grow as they fill, so a large capacity + /// costs nothing up front. + pub capacity_bytes: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SlateDbScanOptions { + /// The bytes a scan fetches from an SST in one request, at least one block. 256KiB by + /// default. + pub read_ahead_bytes: usize, + /// The requests one scan keeps in flight. 4 by default. + pub max_fetch_tasks: usize, + /// Whether the blocks a scan fetches go into the block cache, so that later scans of the same + /// range skip the request. On by default; a large scan then evicts what the cache held. + pub cache_blocks: bool, +} + +impl Default for SlateDbScanOptions { + fn default() -> Self { + Self { + read_ahead_bytes: 256 * 1024, + max_fetch_tasks: 4, + cache_blocks: true, + } + } +} + +impl SlateDbStorage { + pub fn new(db: Arc) -> Self { + Self { + db, + await_durable: true, + scan_options: SlateDbScanOptions::default(), + cache: None, + } + } + + /// Whether a commit waits until its write is durable in object storage (the default). Without + /// it a commit returns once the write is applied in memory, and a crash may lose the latest + /// commits. + pub fn await_durable(mut self, await_durable: bool) -> Self { + self.await_durable = await_durable; + self + } + + /// How scans read SSTs from object storage; see [`SlateDbScanOptions`]. + pub fn scan_options(mut self, scan_options: SlateDbScanOptions) -> Self { + self.scan_options = scan_options; + self + } + + pub fn db(&self) -> &Arc { + &self.db + } + + /// Closes the database, flushing what is still in memory to object storage, and then the + /// cache [`open`](Self::open) built for it. + pub async fn close(&self) -> Result<(), DatabaseError> { + self.db.close().await?; + if let Some(cache) = &self.cache { + cache.close().await?; + } + Ok(()) + } + + pub async fn open( + path: impl Into, + object_store: Arc, + cache: SlateDbCacheOptions, + ) -> Result { + let in_memory = |max_capacity| -> Arc { + Arc::new(FoyerCache::new_with_opts(FoyerCacheOptions { + max_capacity, + ..FoyerCacheOptions::default() + })) + }; + let memory_bytes = cache.memory_bytes.unwrap_or(DEFAULT_BLOCK_CACHE_CAPACITY); + let blocks = match cache.disk { + None if cache.memory_bytes.is_none() => None, + None => Some(in_memory(memory_bytes)), + Some(disk) => { + use foyer::{ + BlockEngineConfig, DeviceBuilder, FsDeviceBuilder, HybridCacheBuilder, + PsyncIoEngineConfig, + }; + let disk_error = |err: foyer::Error| { + DatabaseError::IO(std::io::Error::other(format!("disk cache: {err}"))) + }; + std::fs::create_dir_all(&disk.dir)?; + let device = FsDeviceBuilder::new(&disk.dir) + .with_capacity(disk.capacity_bytes) + .build() + .map_err(disk_error)?; + let hybrid = HybridCacheBuilder::new() + .with_name("kitesql_slatedb_blocks") + .memory(memory_bytes as usize) + .with_weighter(|_, entry: &CachedEntry| entry.size()) + .storage() + .with_io_engine_config(PsyncIoEngineConfig::new()) + .with_engine_config(BlockEngineConfig::new(device)) + .build() + .await + .map_err(disk_error)?; + Some(Arc::new(FoyerHybridCache::new_with_cache(hybrid)) as Arc) + } + }; + // Without its own block cache, the database keeps SlateDB's default one, and closes it. + let db_cache = blocks.map(|blocks| -> Arc { + Arc::new( + SplitCache::new() + .with_block_cache(Some(blocks)) + .with_meta_cache(Some(in_memory(DEFAULT_META_CACHE_CAPACITY))) + .build(), + ) + }); + + let mut builder = Db::builder(path, object_store); + if let Some(db_cache) = &db_cache { + builder = builder.with_db_cache(db_cache.clone(), 0); + } + if cache.cache_compaction_output { + builder = builder.with_block_cache_policy( + BlockCachePolicy::default().with_compaction_output_targets(&[ + CacheTarget::data::<&[u8], _>(..), + CacheTarget::Index, + CacheTarget::Filters, + ]), + ); + } + let mut storage = Self::new(Arc::new(builder.build().await?)); + storage.cache = db_cache; + Ok(storage) + } + + /// [`open`](Self::open)s the SlateDB database at `path` in the S3 bucket `config` describes. + #[cfg(feature = "slatedb-s3")] + pub async fn open_s3( + path: impl Into, + config: &S3Config, + cache: SlateDbCacheOptions, + ) -> Result { + Self::open(path, config.object_store()?, cache).await + } +} + +/// An Amazon S3 bucket, or one of an S3-compatible store such as MinIO or R2. Settings left +/// unset fall back to the standard `AWS_*` environment variables; credentials then fall back to +/// the default AWS chain, such as instance metadata. +#[cfg(feature = "slatedb-s3")] +#[derive(Clone, Default)] +pub struct S3Config { + pub bucket: String, + pub region: Option, + /// The endpoint of an S3-compatible store, such as `http://127.0.0.1:9000` for a local MinIO. + pub endpoint: Option, + pub access_key_id: Option, + pub secret_access_key: Option, + pub session_token: Option, + /// Allows a plain `http://` endpoint. + pub allow_http: bool, +} + +#[cfg(feature = "slatedb-s3")] +impl S3Config { + pub fn new(bucket: impl Into) -> Self { + Self { + bucket: bucket.into(), + ..Self::default() + } + } + + /// Builds the object store for the bucket. It does not connect yet. + pub fn object_store(&self) -> Result, DatabaseError> { + use slatedb::object_store::aws::AmazonS3Builder; + + let mut builder = AmazonS3Builder::from_env().with_bucket_name(&self.bucket); + if let Some(region) = &self.region { + builder = builder.with_region(region); + } + if let Some(endpoint) = &self.endpoint { + builder = builder.with_endpoint(endpoint); + } + if let Some(access_key_id) = &self.access_key_id { + builder = builder.with_access_key_id(access_key_id); + } + if let Some(secret_access_key) = &self.secret_access_key { + builder = builder.with_secret_access_key(secret_access_key); + } + if let Some(session_token) = &self.session_token { + builder = builder.with_token(session_token); + } + if self.allow_http { + builder = builder.with_allow_http(true); + } + Ok(Arc::new(builder.build()?)) + } +} + +#[cfg(feature = "slatedb-s3")] +impl std::fmt::Debug for S3Config { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let redacted = |secret: &Option| secret.as_ref().map(|_| ""); + f.debug_struct("S3Config") + .field("bucket", &self.bucket) + .field("region", &self.region) + .field("endpoint", &self.endpoint) + .field("access_key_id", &self.access_key_id) + .field("secret_access_key", &redacted(&self.secret_access_key)) + .field("session_token", &redacted(&self.session_token)) + .field("allow_http", &self.allow_http) + .finish() + } +} + +impl Storage for SlateDbStorage { + type Metrics = EmptyStorageMetrics; + + type TransactionType<'a> + = SlateDbTransaction + where + Self: 'a; + + fn transaction_with_isolation( + &self, + isolation: TransactionIsolationLevel, + _read_only: bool, + ) -> Result, DatabaseError> { + self.validate_transaction_isolation(isolation)?; + Ok(SlateDbTransaction { + txn: self.db.begin(IsolationLevel::Snapshot)?, + await_durable: self.await_durable, + scan_options: self.scan_options, + statements: 0, + }) + } +} + +pub struct SlateDbTransaction { + txn: DbTransaction, + await_durable: bool, + scan_options: SlateDbScanOptions, + statements: u64, +} + +pub struct SlateDbIter { + iter: DbIterator, + // The entry returned last, which the returned slices borrow. + current: Option, +} + +impl InnerIter for SlateDbIter { + async fn try_next(&mut self) -> Result>, DatabaseError> { + self.current = self.iter.next().await?; + Ok(self + .current + .as_ref() + .map(|entry| (entry.key.as_ref(), entry.value.as_ref()))) + } +} + +impl SlateDbTransaction { + async fn scan( + &self, + min: Bound<&[u8]>, + max: Bound<&[u8]>, + order: IterationOrder, + ) -> Result { + let range = ( + min.map(Bytes::copy_from_slice), + max.map(Bytes::copy_from_slice), + ); + let SlateDbScanOptions { + read_ahead_bytes, + max_fetch_tasks, + cache_blocks, + } = self.scan_options; + let options = ScanOptions { + order, + read_ahead_bytes, + max_fetch_tasks, + cache_blocks, + ..ScanOptions::default() + }; + Ok(SlateDbIter { + iter: self.txn.scan_with_options(range, &options).await?, + current: None, + }) + } +} + +impl StorageTransaction for SlateDbTransaction { + type BorrowedBytes<'a> + = Bytes + where + Self: 'a; + + type IterType<'a> + = SlateDbIter + where + Self: 'a; + + type RevIterType<'a> + = SlateDbIter + where + Self: 'a; + + type PointCursor<'a> + = () + where + Self: 'a; + + fn next_statement_stamp(&mut self) -> Result { + const STATEMENT_BITS: u32 = 12; + self.statements += 1; + let sequence = self.txn.seqnum(); + if self.statements >= 1 << STATEMENT_BITS || sequence >= 1 << (u64::BITS - STATEMENT_BITS) { + return Err(DatabaseError::InvalidValue( + "statement stamps exhausted".into(), + )); + } + Ok((sequence << STATEMENT_BITS) | self.statements) + } + + fn point_cursor(&self) -> Result, DatabaseError> { + Ok(()) + } + + async fn get_borrowed(&self, key: &[u8]) -> Result, DatabaseError> { + Ok(self.txn.get(key).await?) + } + + fn set(&mut self, key: &[u8], value: &[u8]) -> Result<(), DatabaseError> { + Ok(self.txn.put(key, value)?) + } + + fn remove(&mut self, key: &[u8]) -> Result<(), DatabaseError> { + Ok(self.txn.delete(key)?) + } + + async fn range( + &self, + min: Bound<&[u8]>, + max: Bound<&[u8]>, + _pool: &mut BytesPool, + ) -> Result { + self.scan(min, max, IterationOrder::Ascending).await + } + + async fn range_rev( + &self, + min: Bound<&[u8]>, + max: Bound<&[u8]>, + _pool: &mut BytesPool, + ) -> Result { + self.scan(min, max, IterationOrder::Descending).await + } + + async fn commit(self) -> Result<(), DatabaseError> { + let handle = self.txn.commit().await?; + if let (Some(handle), true) = (handle, self.await_durable) { + handle.await_durable().await?; + } + Ok(()) + } +} + +// GRCOV_EXCL_START +#[cfg(test)] +mod tests { + use super::*; + use crate::types::serialize::BytesPool; + use slatedb::object_store::memory::InMemory; + use slatedb::ErrorKind; + + async fn open() -> Result { + let db = Db::open("kitesql-test", Arc::new(InMemory::new())).await?; + Ok(SlateDbStorage::new(Arc::new(db))) + } + + async fn collect(mut iter: SlateDbIter) -> Result>, DatabaseError> { + let mut keys = Vec::new(); + while let Some((key, _)) = iter.try_next().await? { + keys.push(key.to_vec()); + } + Ok(keys) + } + + #[tokio::test] + async fn reads_own_writes_and_commits() -> Result<(), DatabaseError> { + let storage = open().await?; + let mut pool = BytesPool::default(); + let mut txn = storage.transaction()?; + for key in [b"a", b"b", b"c"] { + txn.set(key, b"v")?; + } + txn.remove(b"b")?; + assert_eq!(txn.get_borrowed(b"a").await?.as_deref(), Some(&b"v"[..])); + assert_eq!(txn.get_borrowed(b"b").await?, None); + let iter = txn + .range(Bound::Included(b"a"), Bound::Unbounded, &mut pool) + .await?; + assert_eq!(collect(iter).await?, [b"a".to_vec(), b"c".to_vec()]); + let iter = txn + .range_rev(Bound::Excluded(b"a"), Bound::Included(b"c"), &mut pool) + .await?; + assert_eq!(collect(iter).await?, [b"c".to_vec()]); + txn.commit().await?; + + let txn = storage.transaction()?; + assert_eq!(txn.get_borrowed(b"c").await?.as_deref(), Some(&b"v"[..])); + Ok(()) + } + + // Scans of keys flushed to an SST read them back in order, whatever the read-ahead. + #[tokio::test] + async fn scans_read_flushed_ssts() -> Result<(), DatabaseError> { + let db = Arc::new(Db::open("kitesql-test", Arc::new(InMemory::new())).await?); + let keys = (0..2000u32) + .map(|i| format!("{i:05}").into_bytes()) + .collect::>(); + let mut txn = SlateDbStorage::new(db.clone()).transaction()?; + for key in &keys { + txn.set(key, &[7; 64])?; + } + txn.commit().await?; + db.flush_with_options(slatedb::config::FlushOptions { + flush_type: slatedb::config::FlushType::MemTable, + }) + .await?; + + let mut pool = BytesPool::default(); + for scan_options in [ + SlateDbScanOptions::default(), + SlateDbScanOptions { + read_ahead_bytes: 1, + max_fetch_tasks: 1, + cache_blocks: false, + }, + ] { + let txn = SlateDbStorage::new(db.clone()) + .scan_options(scan_options) + .transaction()?; + let iter = txn + .range(Bound::Unbounded, Bound::Unbounded, &mut pool) + .await?; + assert_eq!(collect(iter).await?, keys); + let iter = txn + .range_rev(Bound::Included(b"01000"), Bound::Unbounded, &mut pool) + .await?; + assert_eq!( + collect(iter).await?, + keys[1000..].iter().rev().cloned().collect::>() + ); + } + Ok(()) + } + + // A database with a disk tier reads back what it wrote, and closes its cache with it. + #[tokio::test] + async fn opens_with_a_disk_cache() -> Result<(), DatabaseError> { + let dir = tempfile::tempdir()?; + let cache = SlateDbCacheOptions { + memory_bytes: Some(1 << 20), + disk: Some(SlateDbDiskCache { + dir: dir.path().join("cache"), + capacity_bytes: 64 << 20, + }), + cache_compaction_output: true, + }; + let storage = + SlateDbStorage::open("kitesql-test", Arc::new(InMemory::new()), cache).await?; + let mut txn = storage.transaction()?; + txn.set(b"k", b"v")?; + txn.commit().await?; + storage + .db() + .flush_with_options(slatedb::config::FlushOptions { + flush_type: slatedb::config::FlushType::MemTable, + }) + .await?; + let txn = storage.transaction()?; + assert_eq!(txn.get_borrowed(b"k").await?.as_deref(), Some(&b"v"[..])); + drop(txn); + storage.close().await?; + assert!(dir.path().join("cache").read_dir()?.next().is_some()); + Ok(()) + } + + #[tokio::test] + async fn remove_range() -> Result<(), DatabaseError> { + let mut transaction = open().await?.await_durable(false).transaction()?; + crate::storage::check_remove_range(&mut transaction).await + } + + #[tokio::test] + async fn range_rev_matches_range() -> Result<(), DatabaseError> { + let mut transaction = open().await?.await_durable(false).transaction()?; + crate::storage::check_range_rev_matches_range(&mut transaction).await + } + + #[tokio::test] + async fn explicit_transaction_does_not_rescan_own_writes() -> Result<(), DatabaseError> { + let storage = open().await?.await_durable(false); + let db = crate::db::DataBaseBuilder::path(".").build_with_storage(storage)?; + crate::storage::check_explicit_transaction_does_not_rescan_own_writes(db).await + } + + #[tokio::test] + async fn dropping_rolls_back() -> Result<(), DatabaseError> { + let storage = open().await?; + let mut txn = storage.transaction()?; + txn.set(b"k", b"v")?; + drop(txn); + assert_eq!(storage.transaction()?.get_borrowed(b"k").await?, None); + Ok(()) + } + + #[tokio::test] + async fn conflicting_commit_fails() -> Result<(), DatabaseError> { + let storage = open().await?.await_durable(false); + let mut first = storage.transaction()?; + let mut second = storage.transaction()?; + first.set(b"k", b"1")?; + second.set(b"k", b"2")?; + first.commit().await?; + assert!(matches!( + second.commit().await, + Err(DatabaseError::SlateDB(err)) if err.kind() == ErrorKind::Transaction + )); + let txn = storage.transaction()?; + assert_eq!(txn.get_borrowed(b"k").await?.as_deref(), Some(&b"1"[..])); + Ok(()) + } + + #[cfg(feature = "slatedb-s3")] + #[test] + fn s3_config_builds_a_store_and_hides_secrets() -> Result<(), DatabaseError> { + let config = S3Config { + endpoint: Some("http://127.0.0.1:9000".into()), + access_key_id: Some("key-id".into()), + secret_access_key: Some("top-secret".into()), + allow_http: true, + ..S3Config::new("kitesql") + }; + config.object_store()?; + let debug = format!("{config:?}"); + assert!(debug.contains("key-id") && !debug.contains("top-secret")); + Ok(()) + } + + #[tokio::test] + async fn statement_stamps_are_distinct() -> Result<(), DatabaseError> { + let storage = open().await?; + let mut txn = storage.transaction()?; + let first = txn.next_statement_stamp()?; + let second = txn.next_statement_stamp()?; + assert!(first != 0 && first != second); + txn.set(b"k", b"v")?; + txn.commit().await?; + // A later transaction starts from a later snapshot, so its stamps differ too. + let mut later = storage.transaction()?; + assert!(![first, second].contains(&later.next_statement_stamp()?)); + Ok(()) + } +} +// GRCOV_EXCL_STOP diff --git a/src/storage/table_codec.rs b/src/storage/table_codec.rs index c3a6eb1d..b2b5a7b3 100644 --- a/src/storage/table_codec.rs +++ b/src/storage/table_codec.rs @@ -55,10 +55,17 @@ static NEXT_TABLE_ID_BYTES: LazyLock> = LazyLock::new(|| b"NextTableId". pub type Bytes = Vec; pub type BumpBytes<'bump> = bumpalo::collections::Vec<'bump, u8>; +#[cfg(not(feature = "async"))] type TupleValueWriter<'a> = ( usize, &'a mut dyn FnMut(&mut Bytes) -> Result<(), DatabaseError>, ); +// The writer is held by the futures of `with_tuple*`, so it must be `Send` on the async path. +#[cfg(feature = "async")] +type TupleValueWriter<'a> = ( + usize, + &'a mut (dyn FnMut(&mut Bytes) -> Result<(), DatabaseError> + Send), +); #[derive(Default)] pub struct TableCodec { @@ -130,6 +137,7 @@ impl IndexStatisticsCodecType { } } +#[maybe_async] impl TableCodec { #[inline] pub(crate) fn bytes_pool(&mut self) -> &mut BytesPool { @@ -144,9 +152,13 @@ impl TableCodec { self.stamp } - pub(crate) fn with_stamp(&mut self, stamp: u64, f: impl FnOnce(&mut Self) -> R) -> R { + pub(crate) async fn with_stamp( + &mut self, + stamp: u64, + f: impl AsyncFnOnce(&mut Self) -> R, + ) -> R { let prev = std::mem::replace(&mut self.stamp, stamp); - let result = f(self); + let result = f(self).await; self.stamp = prev; result } @@ -274,26 +286,27 @@ impl TableCodec { } /// Key: `{TableName}{TUPLE_TAG}{BOUND_MIN_TAG}{RowID}`. - pub fn with_tuple( + pub async fn with_tuple( &mut self, table_id: TableId, tuple_id: &TupleId, tuple_value: Option>, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8]) -> Result, ) -> Result { Self::check_primary_key(tuple_id, 0)?; - self.with_tuple_unchecked(table_id, tuple_id, tuple_value, |key, value, _| { - f(key, value) + self.with_tuple_unchecked(table_id, tuple_id, tuple_value, async |key, value, _| { + f(key, value).await }) + .await } #[inline] - pub(crate) fn with_tuple_unchecked( + pub(crate) async fn with_tuple_unchecked( &mut self, table_id: TableId, tuple_id: &TupleId, tuple_value: Option>, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let prefix = table_id.to_be_bytes(); @@ -309,15 +322,15 @@ impl TableCodec { value.extend_from_slice(&self.stamp.to_le_bytes()); } - f(lower.as_slice(), value.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), value.as_slice(), &mut self.bytes_pool).await } /// Range bounds covering all tuple keys for a table. The bytes pool is /// handed over too, since the codec stays borrowed while `f` runs. - pub(crate) fn with_tuple_bound( + pub(crate) async fn with_tuple_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let prefix = table_id.to_be_bytes(); @@ -328,7 +341,7 @@ impl TableCodec { Self::write_key_prefix(upper, CodecType::Tuple, prefix); upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } /// Key: `{TableName}{INDEX_META_TAG}{BOUND_MIN_TAG}{IndexID}`. @@ -355,10 +368,10 @@ impl TableCodec { } /// Range bounds covering all index metadata for a table. - pub fn with_index_meta_bound( + pub async fn with_index_meta_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let prefix = table_id.to_be_bytes(); @@ -369,15 +382,15 @@ impl TableCodec { Self::write_key_prefix(upper, CodecType::IndexMeta, prefix); upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } /// Range bounds covering a single secondary index. - pub fn with_index_bound( + pub async fn with_index_bound( &mut self, table_id: TableId, index_id: IndexId, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let prefix = table_id.to_be_bytes(); @@ -392,14 +405,14 @@ impl TableCodec { upper.extend_from_slice(&index_id.to_le_bytes()); upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } /// Range bounds covering all secondary indexes for a table. - pub fn with_all_index_bound( + pub async fn with_all_index_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let prefix = table_id.to_be_bytes(); @@ -410,7 +423,7 @@ impl TableCodec { Self::write_key_prefix(upper, CodecType::Index, prefix); upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } /// Non-unique index key: @@ -418,16 +431,18 @@ impl TableCodec { /// /// Unique index key: /// `{TableName}{INDEX_TAG}{BOUND_MIN_TAG}{IndexID}{BOUND_MIN_TAG}{DataValue}` - pub fn with_index( + pub async fn with_index( &mut self, table_id: TableId, index: &Index<'_, V>, tuple_id: Option<&TupleId>, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8]) -> Result, ) -> Result { self.clear_buffers(); let stamp = self.stamp; - self.with_table_buffers(table_id, |lower, prefix, value| { + { + let prefix = table_id.to_be_bytes(); + let (lower, value) = self.slots_mut(); let values_len = (0..index.values.len()) .map(|i| index.values.value_at(i).memcomparable_len_hint()) .sum::(); @@ -459,8 +474,8 @@ impl TableCodec { value.extend_from_slice(&stamp.to_le_bytes()); } - f(lower.as_slice(), value.as_slice()) - }) + f(lower.as_slice(), value.as_slice()).await + } } /// Key: `{TableName}{COLUMN_TAG}{BOUND_MIN_TAG}{ColumnId}`. @@ -493,10 +508,10 @@ impl TableCodec { } /// Range bounds covering all column metadata for a table. - pub fn with_columns_bound( + pub async fn with_columns_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let prefix = table_id.to_be_bytes(); @@ -507,14 +522,14 @@ impl TableCodec { Self::write_key_prefix(upper, CodecType::Column, prefix); upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } /// Range bounds spanning a table's `Column` and `IndexMeta` metadata. - pub fn with_table_bound( + pub async fn with_table_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let prefix = table_id.to_be_bytes(); @@ -525,14 +540,14 @@ impl TableCodec { Self::write_key_prefix(upper, CodecType::IndexMeta, prefix); upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } /// Range bounds covering all statistics keys for a table. - pub fn with_statistics_bound( + pub async fn with_statistics_bound( &mut self, table_id: TableId, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let prefix = table_id.to_be_bytes(); @@ -543,15 +558,15 @@ impl TableCodec { Self::write_key_prefix(upper, CodecType::Statistics, prefix); upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } /// Range bounds covering all statistics keys for one index. - pub fn with_index_statistics_bound( + pub async fn with_index_statistics_bound( &mut self, table_id: TableId, index_id: IndexId, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let prefix = table_id.to_be_bytes(); @@ -565,7 +580,7 @@ impl TableCodec { upper.extend_from_slice(lower); upper.push(BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } fn with_statistics( @@ -577,35 +592,49 @@ impl TableCodec { arena: &(impl MetaArena + ?Sized), f: impl FnOnce(&[u8], &[u8]) -> Result, ) -> Result { + self.encode_statistics(table_id, id, suffix, statistics, arena)?; + let (key, value) = self.slots_mut(); + f(key.as_slice(), value.as_slice()) + } + + fn encode_statistics( + &mut self, + table_id: TableId, + id: &[u8], + suffix: &[u8], + statistics: Option<&V>, + arena: &(impl MetaArena + ?Sized), + ) -> Result<(), DatabaseError> { self.clear_buffers(); - self.with_table_buffers(table_id, |key, prefix, value| { - Self::write_key_prefix(key, CodecType::Statistics, prefix); - key.push(BOUND_MIN_TAG); - key.extend_from_slice(id); - key.extend_from_slice(suffix); - if let Some(statistics) = statistics { - statistics.encode(value, true, arena)?; - } - f(key.as_slice(), value.as_slice()) - }) + let prefix = table_id.to_be_bytes(); + let (key, value) = self.slots_mut(); + Self::write_key_prefix(key, CodecType::Statistics, prefix); + key.push(BOUND_MIN_TAG); + key.extend_from_slice(id); + key.extend_from_slice(suffix); + if let Some(statistics) = statistics { + statistics.encode(value, true, arena)?; + } + Ok(()) } - pub fn with_column_statistics( + pub async fn with_column_statistics( &mut self, table_id: TableId, column_id: ColumnId, meta: Option<&HistogramMeta>, arena: &(impl MetaArena + ?Sized), - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8]) -> Result, ) -> Result { - self.with_statistics( + self.encode_statistics( table_id, &column_id.to_le_bytes(), &[COLUMN_STATISTICS_TAG], meta, arena, - f, - ) + )?; + let (key, value) = self.slots_mut(); + f(key.as_slice(), value.as_slice()).await } pub fn decode_column_statistics( @@ -743,16 +772,18 @@ impl TableCodec { } /// Key: `View{BOUND_MIN_TAG}{ViewID}`. - pub fn with_view( + pub async fn with_view( &mut self, view_id: TableId, - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8]) -> Result, ) -> Result { self.clear_buffers(); - self.with_table_buffers(view_id, |lower, prefix, value| { + { + let prefix = view_id.to_be_bytes(); + let (lower, value) = self.slots_mut(); Self::write_key_prefix(lower, CodecType::View, prefix); - f(lower.as_slice(), value.as_slice()) - }) + f(lower.as_slice(), value.as_slice()).await + } } /// Key: `View{BOUND_MIN_TAG}{ViewID}` with encoded view payload. @@ -772,25 +803,25 @@ impl TableCodec { } /// Range bounds covering all view definitions. - pub fn with_view_bound( + pub async fn with_view_bound( &mut self, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let [lower, upper] = &mut self.buffers; Self::write_global_bound_prefix(lower, VIEW_BYTES.as_slice(), BOUND_MIN_TAG); Self::write_global_bound_prefix(upper, VIEW_BYTES.as_slice(), BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } /// Key: `Root{BOUND_MIN_TAG}{TableName}`, for both tables and views. - pub fn with_root_table( + pub async fn with_root_table( &mut self, table_name: &str, meta: Option<&TableMeta>, arena: &(impl MetaArena + ?Sized), - f: impl FnOnce(&[u8], &[u8]) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8]) -> Result, ) -> Result { self.clear_buffers(); let (lower, value) = self.slots_mut(); @@ -801,28 +832,28 @@ impl TableCodec { Self::encode_root_table_value_into(meta, value, arena)?; } - f(lower.as_slice(), value.as_slice()) + f(lower.as_slice(), value.as_slice()).await } /// Range bounds covering all root table metadata. - pub fn with_root_table_bound( + pub async fn with_root_table_bound( &mut self, - f: impl FnOnce(&[u8], &[u8], &mut BytesPool) -> Result, + f: impl AsyncFnOnce(&[u8], &[u8], &mut BytesPool) -> Result, ) -> Result { self.clear_buffers(); let [lower, upper] = &mut self.buffers; Self::write_global_bound_prefix(lower, ROOT_BYTES.as_slice(), BOUND_MIN_TAG); Self::write_global_bound_prefix(upper, ROOT_BYTES.as_slice(), BOUND_MAX_TAG); - f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool) + f(lower.as_slice(), upper.as_slice(), &mut self.bytes_pool).await } /// Key: `NextTableId`, the id the next table or view gets. - pub fn with_next_table_id( + pub async fn with_next_table_id( &mut self, - f: impl FnOnce(&[u8]) -> Result, + f: impl AsyncFnOnce(&[u8]) -> Result, ) -> Result { - f(NEXT_TABLE_ID_BYTES.as_slice()) + f(NEXT_TABLE_ID_BYTES.as_slice()).await } pub fn decode_tuple_key(bytes: &[u8], pk_ty: &LogicalType) -> Result { @@ -1092,6 +1123,7 @@ impl TableCodec { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use crate::binder::test::build_t1_table; use crate::catalog::view::View; @@ -1105,8 +1137,8 @@ mod tests { }; use crate::optimizer::core::statistics_meta::IndexStatistics; use crate::planner::{PlanArena, TableArenaCell}; - use crate::storage::rocksdb::RocksTransaction; use crate::storage::table_codec::{Bytes, TableCodec}; + use crate::storage::TestTransaction; use crate::types::index::{Index, IndexMeta, IndexType}; use crate::types::tuple::Tuple; use crate::types::value::{DataValue, OrderedMapping, TupleMappingRef, Utf8Type}; @@ -1135,8 +1167,8 @@ mod tests { } #[test] - fn covered_key_decode_uses_skip_configuration_and_output_mapping() -> Result<(), DatabaseError> - { + async fn covered_key_decode_uses_skip_configuration_and_output_mapping( + ) -> Result<(), DatabaseError> { let varchar = LogicalType::Varchar(None, CharLengthUnits::Characters); let char_type = LogicalType::Char(16, CharLengthUnits::Octets); let values = vec![ @@ -1156,10 +1188,11 @@ mod tests { ]; let index = Index::new(7, values.as_slice(), IndexType::Normal); let mut codec = TableCodec::default(); - let (key, value) = - codec.with_index(0, &index, Some(&DataValue::Int32(1)), |key, value| { + let (key, value) = codec + .with_index(0, &index, Some(&DataValue::Int32(1)), async |key, value| { Ok((key.to_vec(), value.to_vec())) - })?; + }) + .await?; let index_ty = LogicalType::Tuple(vec![ LogicalType::Integer, varchar.clone(), @@ -1210,7 +1243,7 @@ mod tests { } #[test] - fn test_table_codec_tuple() -> Result<(), DatabaseError> { + async fn test_table_codec_tuple() -> Result<(), DatabaseError> { let table_arena = TableArenaCell::default(); let table_catalog = build_table_codec(&table_arena); let plan_arena = PlanArena::new(&table_arena); @@ -1224,12 +1257,14 @@ mod tests { LogicalType::Decimal(None, None).serializable(), ]; let mut write_value = |value: &mut Bytes| expected.serialize_to(&serializers, value); - let bytes = TableCodec::default().with_tuple( - 0, - &DataValue::Int32(0), - Some((expected.serialized_len_hint(), &mut write_value)), - |_, value| Ok(value.to_vec()), - )?; + let bytes = TableCodec::default() + .with_tuple( + 0, + &DataValue::Int32(0), + Some((expected.serialized_len_hint(), &mut write_value)), + async |_, value| Ok(value.to_vec()), + ) + .await?; let deserializers = table_catalog .columns() .map(|column| plan_arena.column(*column).datatype().serializable()) @@ -1260,7 +1295,7 @@ mod tests { } #[test] - fn test_root_catalog() { + async fn test_root_catalog() { let mut table_codec = TableCodec::default(); let table_arena = TableArenaCell::default(); let table_catalog = build_table_codec(&table_arena); @@ -1270,19 +1305,20 @@ mod tests { table_catalog.name.as_ref(), Some(&meta), table_arena.borrow(), - |_, value| Ok::<_, DatabaseError>(value.to_vec()), + async |_, value| Ok::<_, DatabaseError>(value.to_vec()), ) + .await .unwrap(); let table_meta = - TableCodec::decode_root_table::(&bytes, table_arena.borrow_mut()) + TableCodec::decode_root_table::(&bytes, table_arena.borrow_mut()) .unwrap(); assert_eq!(table_meta, meta); } #[test] - fn test_table_codec_column_and_index_statistics() -> Result<(), DatabaseError> { + async fn test_table_codec_column_and_index_statistics() -> Result<(), DatabaseError> { let mut table_codec = TableCodec::default(); let table_arena = TableArenaCell::default(); let mut plan_arena = PlanArena::new(&table_arena); @@ -1313,7 +1349,7 @@ mod tests { Ok::<_, DatabaseError>(value.to_vec()) })?; let decoded_root = - TableCodec::decode_index_statistics::(&root_bytes, &mut plan_arena)?; + TableCodec::decode_index_statistics::(&root_bytes, &mut plan_arena)?; assert_eq!(decoded_root.index_id(), root.index_id()); let index_key = table_codec .with_index_statistics(0, 0, None, &plan_arena, |key, _| Ok(key.to_vec()))?; @@ -1327,26 +1363,28 @@ mod tests { builder.append(&value); } let column_meta = builder.build(); - let (column_key, bytes) = table_codec.with_column_statistics( - 0, - 0, - Some(&column_meta), - &plan_arena, - |key, value| Ok((key.to_vec(), value.to_vec())), - )?; + let (column_key, bytes) = table_codec + .with_column_statistics(0, 0, Some(&column_meta), &plan_arena, async |key, value| { + Ok((key.to_vec(), value.to_vec())) + }) + .await?; assert_ne!(column_key, index_key); - table_codec.with_index_statistics_bound(0, 0, |min, max, _| { - assert!(index_key.as_slice() >= min && index_key.as_slice() <= max); - assert!(column_key.as_slice() < min || column_key.as_slice() > max); - Ok(()) - })?; - table_codec.with_statistics_bound(0, |min, max, _| { - assert!(index_key.as_slice() >= min && index_key.as_slice() <= max); - assert!(column_key.as_slice() >= min && column_key.as_slice() <= max); - Ok(()) - })?; + table_codec + .with_index_statistics_bound(0, 0, async |min, max, _| { + assert!(index_key.as_slice() >= min && index_key.as_slice() <= max); + assert!(column_key.as_slice() < min || column_key.as_slice() > max); + Ok(()) + }) + .await?; + table_codec + .with_statistics_bound(0, async |min, max, _| { + assert!(index_key.as_slice() >= min && index_key.as_slice() <= max); + assert!(column_key.as_slice() >= min && column_key.as_slice() <= max); + Ok(()) + }) + .await?; let decoded = - TableCodec::decode_column_statistics::(&bytes, &mut plan_arena)?; + TableCodec::decode_column_statistics::(&bytes, &mut plan_arena)?; assert_eq!(decoded, column_meta); assert_eq!(decoded.distinct_values_len(), 2); assert_eq!(decoded.null_count(), 1); @@ -1368,9 +1406,10 @@ mod tests { &plan_arena, |_, value| Ok::<_, DatabaseError>(value.to_vec()), )?; - let decoded_sketch_meta = TableCodec::decode_index_statistics_sketch_meta::< - RocksTransaction, - >(&sketch_meta_bytes, &mut plan_arena)?; + let decoded_sketch_meta = TableCodec::decode_index_statistics_sketch_meta::( + &sketch_meta_bytes, + &mut plan_arena, + )?; assert_eq!(decoded_sketch_meta.width(), sketch_meta.width()); assert_eq!(decoded_sketch_meta.k_num(), sketch_meta.k_num()); @@ -1383,9 +1422,10 @@ mod tests { &plan_arena, |_, value| Ok::<_, DatabaseError>(value.to_vec()), )?; - let decoded_sketch_page = TableCodec::decode_index_statistics_sketch_page::< - RocksTransaction, - >(&sketch_page_bytes, &mut plan_arena)?; + let decoded_sketch_page = TableCodec::decode_index_statistics_sketch_page::( + &sketch_page_bytes, + &mut plan_arena, + )?; assert_eq!(decoded_sketch_page.counters(), first_sketch_page.counters()); let bucket0_key = @@ -1398,10 +1438,11 @@ mod tests { })?; assert!(bucket0_key < bucket1_key); - let (bucket0_min, bucket0_max) = - table_codec.with_index_statistics_bound(0, 0, |min, max, _| { + let (bucket0_min, bucket0_max) = table_codec + .with_index_statistics_bound(0, 0, async |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) - })?; + }) + .await?; assert!(bucket0_key.as_slice() >= bucket0_min.as_slice()); assert!(bucket1_key.as_slice() <= bucket0_max.as_slice()); @@ -1413,7 +1454,7 @@ mod tests { &plan_arena, |_, value| Ok::<_, DatabaseError>(value.to_vec()), )?; - let decoded_bucket = TableCodec::decode_index_statistics_bucket::( + let decoded_bucket = TableCodec::decode_index_statistics_bucket::( &bucket_bytes, &mut plan_arena, )?; @@ -1441,7 +1482,7 @@ mod tests { &plan_arena, |_, value| Ok::<_, DatabaseError>(value.to_vec()), )?; - let decoded_top_n = TableCodec::decode_index_statistics_top_n::( + let decoded_top_n = TableCodec::decode_index_statistics_top_n::( &top_n_bytes, &mut plan_arena, )?; @@ -1470,7 +1511,7 @@ mod tests { })?; assert_eq!( - TableCodec::decode_index_meta::(&bytes, &mut plan_arena)?, + TableCodec::decode_index_meta::(&bytes, &mut plan_arena)?, index_meta ); @@ -1478,11 +1519,14 @@ mod tests { } #[test] - fn test_table_codec_index() -> Result<(), DatabaseError> { + async fn test_table_codec_index() -> Result<(), DatabaseError> { let tuple_id = DataValue::Int32(0); let index = Index::new(0, &[][..], IndexType::Unique); let bytes = TableCodec::default() - .with_index(0, &index, Some(&tuple_id), |_, value| Ok(value.to_vec()))?; + .with_index(0, &index, Some(&tuple_id), async |_, value| { + Ok(value.to_vec()) + }) + .await?; assert_eq!(TableCodec::decode_index(&bytes)?, tuple_id); @@ -1512,7 +1556,7 @@ mod tests { .unwrap(); let mut cursor = Cursor::new(bytes); let decode_col = - TableCodec::decode_column::(&mut cursor, &mut plan_arena)?; + TableCodec::decode_column::(&mut cursor, &mut plan_arena)?; assert_eq!(decode_col, expected_col); @@ -1520,7 +1564,7 @@ mod tests { } #[test] - fn test_table_codec_view() -> Result<(), DatabaseError> { + async fn test_table_codec_view() -> Result<(), DatabaseError> { fn normalize_explain(explain: String) -> String { let mut normalized = String::with_capacity(explain.len()); let bytes = explain.as_bytes(); @@ -1550,7 +1594,7 @@ mod tests { } let mut table_codec = TableCodec::default(); - let table_state = build_t1_table()?; + let table_state = build_t1_table().await?; let scala_functions = Default::default(); let table_functions = Default::default(); let build_view = |name: &str, sql: &str| -> Result<(View, PlanArena<'_>), DatabaseError> { @@ -1576,7 +1620,7 @@ mod tests { println!("{:#?}", view.plan); let mut decode_arena = PlanArena::new(&table_state.table_arena); let decoded = table_codec.with_view_value(0, &view, &plan_arena, |_, value| { - TableCodec::decode_view::( + TableCodec::decode_view::( value, &scala_functions, &table_functions, @@ -1597,7 +1641,7 @@ mod tests { build_view("view_filter", "select * from t1 where c1 > 1")?; let mut decode_arena = PlanArena::new(&table_state.table_arena); let decoded = table_codec.with_view_value(0, &view, &plan_arena, |_, value| { - TableCodec::decode_view::( + TableCodec::decode_view::( value, &scala_functions, &table_functions, @@ -1618,7 +1662,7 @@ mod tests { build_view("view_join", "select * from t1 left join t2 on c1 = c3")?; let mut decode_arena = PlanArena::new(&table_state.table_arena); let decoded = table_codec.with_view_value(0, &view, &plan_arena, |_, value| { - TableCodec::decode_view::( + TableCodec::decode_view::( value, &scala_functions, &table_functions, @@ -1638,7 +1682,7 @@ mod tests { #[test] #[allow(clippy::mutable_key_type)] - fn test_table_codec_column_bound() { + async fn test_table_codec_column_bound() { let mut table_codec = TableCodec::default(); let mut set = BTreeSet::new(); let op = |col_id: usize, table_id: TableId| { @@ -1676,9 +1720,10 @@ mod tests { set.insert(op(0, 2)); let (min, max) = table_codec - .with_columns_bound(1, |min, max, _| { + .with_columns_bound(1, async |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) + .await .unwrap(); let vec = set @@ -1697,7 +1742,7 @@ mod tests { #[test] #[allow(clippy::mutable_key_type)] - fn test_table_codec_index_meta_bound() { + async fn test_table_codec_index_meta_bound() { let mut table_codec = TableCodec::default(); let mut set = BTreeSet::new(); let op = |index_id: usize, table_id: TableId| { @@ -1734,9 +1779,10 @@ mod tests { set.insert(op(2, 2)); let (min, max) = table_codec - .with_index_meta_bound(1, |min, max, _| { + .with_index_meta_bound(1, async |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) + .await .unwrap(); let vec = set @@ -1755,7 +1801,7 @@ mod tests { #[test] #[allow(clippy::mutable_key_type)] - fn test_table_codec_index_bound() { + async fn test_table_codec_index_bound() { let mut table_codec = TableCodec::default(); let mut set = BTreeSet::new(); let column = ColumnCatalog::new( @@ -1767,7 +1813,7 @@ mod tests { let table_catalog = TableCatalog::new(0, "T0".to_string(), vec![column], table_arena.borrow_mut()).unwrap(); - let op = |value: DataValue, index_id: usize, table_id: TableId| { + let op = async |value: DataValue, index_id: usize, table_id: TableId| { let mut table_codec = TableCodec::default(); let value = Arc::new(value); let index = Index::new( @@ -1777,28 +1823,30 @@ mod tests { ); table_codec - .with_index(table_id, &index, None, |key, _| { + .with_index(table_id, &index, None, async |key, _| { Ok::<_, DatabaseError>(key.to_vec()) }) + .await .unwrap() }; - set.insert(op(DataValue::Int32(0), 0, table_catalog.id())); - set.insert(op(DataValue::Int32(1), 0, table_catalog.id())); - set.insert(op(DataValue::Int32(2), 0, table_catalog.id())); + set.insert(op(DataValue::Int32(0), 0, table_catalog.id()).await); + set.insert(op(DataValue::Int32(1), 0, table_catalog.id()).await); + set.insert(op(DataValue::Int32(2), 0, table_catalog.id()).await); - set.insert(op(DataValue::Int32(0), 1, table_catalog.id())); - set.insert(op(DataValue::Int32(1), 1, table_catalog.id())); - set.insert(op(DataValue::Int32(2), 1, table_catalog.id())); + set.insert(op(DataValue::Int32(0), 1, table_catalog.id()).await); + set.insert(op(DataValue::Int32(1), 1, table_catalog.id()).await); + set.insert(op(DataValue::Int32(2), 1, table_catalog.id()).await); - set.insert(op(DataValue::Int32(0), 2, table_catalog.id())); - set.insert(op(DataValue::Int32(1), 2, table_catalog.id())); - set.insert(op(DataValue::Int32(2), 2, table_catalog.id())); + set.insert(op(DataValue::Int32(0), 2, table_catalog.id()).await); + set.insert(op(DataValue::Int32(1), 2, table_catalog.id()).await); + set.insert(op(DataValue::Int32(2), 2, table_catalog.id()).await); let (min, max) = table_codec - .with_index_bound(table_catalog.id(), 1, |min, max, _| { + .with_index_bound(table_catalog.id(), 1, async |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) + .await .unwrap(); let vec = set @@ -1810,17 +1858,26 @@ mod tests { assert_eq!(vec.len(), 3); - assert_eq!(vec[0], &op(DataValue::Int32(0), 1, table_catalog.id())); - assert_eq!(vec[1], &op(DataValue::Int32(1), 1, table_catalog.id())); - assert_eq!(vec[2], &op(DataValue::Int32(2), 1, table_catalog.id())); + assert_eq!( + vec[0], + &op(DataValue::Int32(0), 1, table_catalog.id()).await + ); + assert_eq!( + vec[1], + &op(DataValue::Int32(1), 1, table_catalog.id()).await + ); + assert_eq!( + vec[2], + &op(DataValue::Int32(2), 1, table_catalog.id()).await + ); } #[test] #[allow(clippy::mutable_key_type)] - fn test_table_codec_index_all_bound() { + async fn test_table_codec_index_all_bound() { let mut table_codec = TableCodec::default(); let mut set = BTreeSet::new(); - let op = |value: DataValue, index_id: usize, table_id: TableId| { + let op = async |value: DataValue, index_id: usize, table_id: TableId| { let mut table_codec = TableCodec::default(); let value = Arc::new(value); let index = Index::new( @@ -1830,28 +1887,30 @@ mod tests { ); table_codec - .with_index(table_id, &index, None, |key, _| { + .with_index(table_id, &index, None, async |key, _| { Ok::<_, DatabaseError>(key.to_vec()) }) + .await .unwrap() }; - set.insert(op(DataValue::Int32(0), 0, 0)); - set.insert(op(DataValue::Int32(1), 0, 0)); - set.insert(op(DataValue::Int32(2), 0, 0)); + set.insert(op(DataValue::Int32(0), 0, 0).await); + set.insert(op(DataValue::Int32(1), 0, 0).await); + set.insert(op(DataValue::Int32(2), 0, 0).await); - set.insert(op(DataValue::Int32(0), 0, 1)); - set.insert(op(DataValue::Int32(1), 0, 1)); - set.insert(op(DataValue::Int32(2), 0, 1)); + set.insert(op(DataValue::Int32(0), 0, 1).await); + set.insert(op(DataValue::Int32(1), 0, 1).await); + set.insert(op(DataValue::Int32(2), 0, 1).await); - set.insert(op(DataValue::Int32(0), 0, 2)); - set.insert(op(DataValue::Int32(1), 0, 2)); - set.insert(op(DataValue::Int32(2), 0, 2)); + set.insert(op(DataValue::Int32(0), 0, 2).await); + set.insert(op(DataValue::Int32(1), 0, 2).await); + set.insert(op(DataValue::Int32(2), 0, 2).await); let (min, max) = table_codec - .with_all_index_bound(1, |min, max, _| { + .with_all_index_bound(1, async |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) + .await .unwrap(); let vec = set @@ -1863,41 +1922,43 @@ mod tests { assert_eq!(vec.len(), 3); - assert_eq!(vec[0], &op(DataValue::Int32(0), 0, 1)); - assert_eq!(vec[1], &op(DataValue::Int32(1), 0, 1)); - assert_eq!(vec[2], &op(DataValue::Int32(2), 0, 1)); + assert_eq!(vec[0], &op(DataValue::Int32(0), 0, 1).await); + assert_eq!(vec[1], &op(DataValue::Int32(1), 0, 1).await); + assert_eq!(vec[2], &op(DataValue::Int32(2), 0, 1).await); } #[test] #[allow(clippy::mutable_key_type)] - fn test_table_codec_tuple_bound() { + async fn test_table_codec_tuple_bound() { let mut table_codec = TableCodec::default(); let mut set = BTreeSet::new(); - let op = |tuple_id: DataValue, table_id: TableId| { + let op = async |tuple_id: DataValue, table_id: TableId| { let mut table_codec = TableCodec::default(); table_codec - .with_tuple(table_id, &Arc::new(tuple_id), None, |key, _| { + .with_tuple(table_id, &Arc::new(tuple_id), None, async |key, _| { Ok::<_, DatabaseError>(key.to_vec()) }) + .await .unwrap() }; - set.insert(op(DataValue::Int32(0), 0)); - set.insert(op(DataValue::Int32(1), 0)); - set.insert(op(DataValue::Int32(2), 0)); + set.insert(op(DataValue::Int32(0), 0).await); + set.insert(op(DataValue::Int32(1), 0).await); + set.insert(op(DataValue::Int32(2), 0).await); - set.insert(op(DataValue::Int32(0), 1)); - set.insert(op(DataValue::Int32(1), 1)); - set.insert(op(DataValue::Int32(2), 1)); + set.insert(op(DataValue::Int32(0), 1).await); + set.insert(op(DataValue::Int32(1), 1).await); + set.insert(op(DataValue::Int32(2), 1).await); - set.insert(op(DataValue::Int32(0), 2)); - set.insert(op(DataValue::Int32(1), 2)); - set.insert(op(DataValue::Int32(2), 2)); + set.insert(op(DataValue::Int32(0), 2).await); + set.insert(op(DataValue::Int32(1), 2).await); + set.insert(op(DataValue::Int32(2), 2).await); let (min, max) = table_codec - .with_tuple_bound(1, |min, max, _| { + .with_tuple_bound(1, async |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) + .await .unwrap(); let vec = set @@ -1909,24 +1970,25 @@ mod tests { assert_eq!(vec.len(), 3); - assert_eq!(vec[0], &op(DataValue::Int32(0), 1)); - assert_eq!(vec[1], &op(DataValue::Int32(1), 1)); - assert_eq!(vec[2], &op(DataValue::Int32(2), 1)); + assert_eq!(vec[0], &op(DataValue::Int32(0), 1).await); + assert_eq!(vec[1], &op(DataValue::Int32(1), 1).await); + assert_eq!(vec[2], &op(DataValue::Int32(2), 1).await); } #[test] #[allow(clippy::mutable_key_type)] - fn test_root_codec_name_bound() { + async fn test_root_codec_name_bound() { let mut table_codec = TableCodec::default(); let mut set: BTreeSet = BTreeSet::new(); - let op = |table_name: &str| { + let op = async |table_name: &str| { let mut table_codec = TableCodec::default(); let table_arena = TableArenaCell::default(); let plan_arena = PlanArena::new(&table_arena); table_codec - .with_root_table(table_name, None, &plan_arena, |key, _| { + .with_root_table(table_name, None, &plan_arena, async |key, _| { Ok::<_, DatabaseError>(key.to_vec()) }) + .await .unwrap() }; @@ -1935,14 +1997,15 @@ mod tests { set.insert(value_0); set.insert(value_1); - set.insert(op("T0")); - set.insert(op("T1")); - set.insert(op("T2")); + set.insert(op("T0").await); + set.insert(op("T1").await); + set.insert(op("T2").await); let (min, max) = table_codec - .with_root_table_bound(|min, max, _| { + .with_root_table_bound(async |min, max, _| { Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) }) + .await .unwrap(); let vec = set @@ -1952,20 +2015,21 @@ mod tests { )) .collect_vec(); - assert_eq!(vec[0], &op("T0")); - assert_eq!(vec[1], &op("T1")); - assert_eq!(vec[2], &op("T2")); + assert_eq!(vec[0], &op("T0").await); + assert_eq!(vec[1], &op("T1").await); + assert_eq!(vec[2], &op("T2").await); } #[test] #[allow(clippy::mutable_key_type)] - fn test_view_codec_name_bound() { + async fn test_view_codec_name_bound() { let mut table_codec = TableCodec::default(); let mut set = BTreeSet::new(); - let op = |view_id: TableId| { + let op = async |view_id: TableId| { let mut table_codec = TableCodec::default(); table_codec - .with_view(view_id, |key, _| Ok::<_, DatabaseError>(key.to_vec())) + .with_view(view_id, async |key, _| Ok::<_, DatabaseError>(key.to_vec())) + .await .unwrap() }; @@ -1975,12 +2039,15 @@ mod tests { set.insert(value_0); set.insert(value_1); - set.insert(op(0)); - set.insert(op(1)); - set.insert(op(2)); + set.insert(op(0).await); + set.insert(op(1).await); + set.insert(op(2).await); let (min, max) = table_codec - .with_view_bound(|min, max, _| Ok::<_, DatabaseError>((min.to_vec(), max.to_vec()))) + .with_view_bound(async |min, max, _| { + Ok::<_, DatabaseError>((min.to_vec(), max.to_vec())) + }) + .await .unwrap(); let vec = set @@ -1990,9 +2057,9 @@ mod tests { )) .collect_vec(); - assert_eq!(vec[0], &op(0)); - assert_eq!(vec[1], &op(1)); - assert_eq!(vec[2], &op(2)); + assert_eq!(vec[0], &op(0).await); + assert_eq!(vec[1], &op(1).await); + assert_eq!(vec[2], &op(2).await); } } // GRCOV_EXCL_STOP diff --git a/src/types/bitset.rs b/src/types/bitset.rs index 295b7052..4849dc1c 100644 --- a/src/types/bitset.rs +++ b/src/types/bitset.rs @@ -63,6 +63,7 @@ impl BitSet { // GRCOV_EXCL_START #[cfg(test)] +#[maybe_async] mod tests { use super::BitSet; diff --git a/src/types/evaluator/binary.rs b/src/types/evaluator/binary.rs index 4216169f..2421a64d 100644 --- a/src/types/evaluator/binary.rs +++ b/src/types/evaluator/binary.rs @@ -701,12 +701,13 @@ macro_rules! numeric_binary_evaluator_definition { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::errors::DatabaseError; use crate::expression::BinaryOperator; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::evaluator::BinaryEvaluatorRef; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -846,7 +847,7 @@ mod test { cursor.seek(SeekFrom::Start(0))?; assert_eq!( - BinaryEvaluatorRef::decode::(&mut cursor, None, &mut arena,)?, + BinaryEvaluatorRef::decode::(&mut cursor, None, &mut arena,)?, evaluator ); diff --git a/src/types/evaluator/boolean.rs b/src/types/evaluator/boolean.rs index bf1a1e73..ba92b55e 100644 --- a/src/types/evaluator/boolean.rs +++ b/src/types/evaluator/boolean.rs @@ -129,6 +129,7 @@ pub fn boolean_not_eq_binary_eval( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/cast.rs b/src/types/evaluator/cast.rs index 56be76c0..fa109f12 100644 --- a/src/types/evaluator/cast.rs +++ b/src/types/evaluator/cast.rs @@ -1225,11 +1225,12 @@ impl CastEvaluatorRef { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::errors::DatabaseError; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::evaluator::CastEvaluatorRef; use crate::types::value::{DataValue, Utf8Type}; use crate::types::{CharLengthUnits, LogicalType}; @@ -1385,7 +1386,7 @@ mod test { cursor.seek(SeekFrom::Start(0))?; assert_eq!( - CastEvaluatorRef::decode::(&mut cursor, None, &mut arena)?, + CastEvaluatorRef::decode::(&mut cursor, None, &mut arena)?, evaluator ); diff --git a/src/types/evaluator/date.rs b/src/types/evaluator/date.rs index 98fe5152..929da4cf 100644 --- a/src/types/evaluator/date.rs +++ b/src/types/evaluator/date.rs @@ -75,6 +75,7 @@ crate::define_cast_evaluator!(date32_to_datetime_cast_eval, DataValue::Date32(va // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::errors::DatabaseError; diff --git a/src/types/evaluator/datetime.rs b/src/types/evaluator/datetime.rs index acfd753d..5850c737 100644 --- a/src/types/evaluator/datetime.rs +++ b/src/types/evaluator/datetime.rs @@ -98,6 +98,7 @@ crate::define_cast_evaluator!( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::errors::DatabaseError; diff --git a/src/types/evaluator/decimal.rs b/src/types/evaluator/decimal.rs index f1d0c293..6aa1e979 100644 --- a/src/types/evaluator/decimal.rs +++ b/src/types/evaluator/decimal.rs @@ -143,6 +143,7 @@ decimal_binary!( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/float32.rs b/src/types/evaluator/float32.rs index efb224f9..70285ec2 100644 --- a/src/types/evaluator/float32.rs +++ b/src/types/evaluator/float32.rs @@ -112,6 +112,7 @@ crate::define_float_cast_evaluators!(Float32, Float32, f32, LogicalType::Float, // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/float64.rs b/src/types/evaluator/float64.rs index 260e3d4e..946b3035 100644 --- a/src/types/evaluator/float64.rs +++ b/src/types/evaluator/float64.rs @@ -184,6 +184,7 @@ float64_binary!( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/int16.rs b/src/types/evaluator/int16.rs index 17cc8e8b..c372f2ac 100644 --- a/src/types/evaluator/int16.rs +++ b/src/types/evaluator/int16.rs @@ -22,6 +22,7 @@ crate::define_integer_cast_evaluators!(Int16, Int16, i16, LogicalType::Smallint) // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/int32.rs b/src/types/evaluator/int32.rs index 10f637ba..694be8de 100644 --- a/src/types/evaluator/int32.rs +++ b/src/types/evaluator/int32.rs @@ -22,6 +22,7 @@ crate::define_integer_cast_evaluators!(Int32, Int32, i32, LogicalType::Integer); // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/int64.rs b/src/types/evaluator/int64.rs index ce26e48e..e51b1aa3 100644 --- a/src/types/evaluator/int64.rs +++ b/src/types/evaluator/int64.rs @@ -22,6 +22,7 @@ crate::define_integer_cast_evaluators!(Int64, Int64, i64, LogicalType::Bigint); // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/int8.rs b/src/types/evaluator/int8.rs index 9aad2190..63f6396a 100644 --- a/src/types/evaluator/int8.rs +++ b/src/types/evaluator/int8.rs @@ -22,6 +22,7 @@ crate::define_integer_cast_evaluators!(Int8, Int8, i8, LogicalType::Tinyint); // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/null.rs b/src/types/evaluator/null.rs index c1a14760..dc9cae8f 100644 --- a/src/types/evaluator/null.rs +++ b/src/types/evaluator/null.rs @@ -29,6 +29,7 @@ pub fn null_cast_eval(_value: DataValue) -> Result { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; diff --git a/src/types/evaluator/time32.rs b/src/types/evaluator/time32.rs index 906f9efc..ab370e1f 100644 --- a/src/types/evaluator/time32.rs +++ b/src/types/evaluator/time32.rs @@ -160,6 +160,7 @@ pub fn time_not_eq_binary_eval( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/time64.rs b/src/types/evaluator/time64.rs index f0fdcc06..ebff2a78 100644 --- a/src/types/evaluator/time64.rs +++ b/src/types/evaluator/time64.rs @@ -171,6 +171,7 @@ time64_binary!(time64_not_eq_binary_eval, !=); // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/tuple.rs b/src/types/evaluator/tuple.rs index c276b6e9..f34833d7 100644 --- a/src/types/evaluator/tuple.rs +++ b/src/types/evaluator/tuple.rs @@ -121,6 +121,7 @@ pub(crate) fn eval_tuple_cast( // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::evaluator::cast_create; diff --git a/src/types/evaluator/uint16.rs b/src/types/evaluator/uint16.rs index 856e44c5..7a89a697 100644 --- a/src/types/evaluator/uint16.rs +++ b/src/types/evaluator/uint16.rs @@ -21,6 +21,7 @@ crate::define_integer_cast_evaluators!(Uint16, UInt16, u16, LogicalType::USmalli // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/uint32.rs b/src/types/evaluator/uint32.rs index f93f5b09..ecd81493 100644 --- a/src/types/evaluator/uint32.rs +++ b/src/types/evaluator/uint32.rs @@ -21,6 +21,7 @@ crate::define_integer_cast_evaluators!(Uint32, UInt32, u32, LogicalType::UIntege // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/uint64.rs b/src/types/evaluator/uint64.rs index 3c2442f3..f5918da7 100644 --- a/src/types/evaluator/uint64.rs +++ b/src/types/evaluator/uint64.rs @@ -21,6 +21,7 @@ crate::define_integer_cast_evaluators!(Uint64, UInt64, u64, LogicalType::UBigint // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/uint8.rs b/src/types/evaluator/uint8.rs index 9521df67..dd923d33 100644 --- a/src/types/evaluator/uint8.rs +++ b/src/types/evaluator/uint8.rs @@ -21,6 +21,7 @@ crate::define_integer_cast_evaluators!(Uint8, UInt8, u8, LogicalType::UTinyint); // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::types::value::Utf8Type; diff --git a/src/types/evaluator/unary.rs b/src/types/evaluator/unary.rs index 315c6617..f12f20ea 100644 --- a/src/types/evaluator/unary.rs +++ b/src/types/evaluator/unary.rs @@ -94,12 +94,13 @@ macro_rules! numeric_unary_evaluator_definition { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::errors::DatabaseError; use crate::expression::UnaryOperator; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::evaluator::UnaryEvaluatorRef; use crate::types::value::DataValue; use crate::types::LogicalType; @@ -252,7 +253,7 @@ mod test { cursor.seek(SeekFrom::Start(0))?; assert_eq!( - UnaryEvaluatorRef::decode::(&mut cursor, None, &mut arena)?, + UnaryEvaluatorRef::decode::(&mut cursor, None, &mut arena)?, evaluator ); diff --git a/src/types/evaluator/utf8.rs b/src/types/evaluator/utf8.rs index a5890f61..ab5259ba 100644 --- a/src/types/evaluator/utf8.rs +++ b/src/types/evaluator/utf8.rs @@ -353,6 +353,7 @@ fn next_char_at(input: &str, index: usize) -> Option<(char, usize)> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use super::*; use crate::expression::BinaryOperator; diff --git a/src/types/index.rs b/src/types/index.rs index afc93064..96460ad9 100644 --- a/src/types/index.rs +++ b/src/types/index.rs @@ -210,12 +210,13 @@ impl fmt::Display for IndexMetaRef { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; use crate::catalog::{ColumnCatalog, ColumnDesc, TableCatalog}; use crate::planner::{PlanArena, TableArena, TableArenaCell}; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use std::fmt::Debug; use std::io::{Cursor, Seek, SeekFrom}; @@ -257,7 +258,7 @@ mod tests { value.encode(&mut cursor, false, encode_arena)?; cursor.seek(SeekFrom::Start(0))?; - T::decode::(&mut cursor, None, decode_arena) + T::decode::(&mut cursor, None, decode_arena) } #[test] diff --git a/src/types/mod.rs b/src/types/mod.rs index 1ef0fa12..77fafe59 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -505,10 +505,11 @@ impl std::fmt::Display for LogicalType { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] pub(crate) mod test { use crate::errors::DatabaseError; use crate::serdes::ReferenceSerialization; - use crate::storage::rocksdb::RocksTransaction; + use crate::storage::TestTransaction; use crate::types::CharLengthUnits; use crate::types::LogicalType; #[cfg(feature = "time")] @@ -994,7 +995,7 @@ pub(crate) mod test { cursor.seek(SeekFrom::Start(0))?; assert_eq!( - LogicalType::decode::(cursor, None, &mut arena,)?, + LogicalType::decode::(cursor, None, &mut arena,)?, logical_type ); cursor.seek(SeekFrom::Start(0))?; diff --git a/src/types/serialize.rs b/src/types/serialize.rs index 030860c7..745176a1 100644 --- a/src/types/serialize.rs +++ b/src/types/serialize.rs @@ -843,6 +843,7 @@ impl LogicalType { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 2e0a0bed..3a0cd267 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -813,6 +813,7 @@ impl Tuple { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::TupleLike; use crate::catalog::{ColumnCatalog, ColumnDesc}; diff --git a/src/types/tuple_builder.rs b/src/types/tuple_builder.rs index ba957232..d93d2703 100644 --- a/src/types/tuple_builder.rs +++ b/src/types/tuple_builder.rs @@ -83,6 +83,7 @@ impl<'a> TupleBuilder<'a> { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod tests { use super::*; diff --git a/src/types/value.rs b/src/types/value.rs index 2202fee4..85d0e03b 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -2196,6 +2196,7 @@ impl fmt::Debug for DataValue { // GRCOV_EXCL_START #[cfg(all(test, not(target_arch = "wasm32")))] +#[maybe_async] mod test { use crate::errors::DatabaseError; use crate::storage::table_codec::{BumpBytes, NOTNULL_TAG, NULL_TAG}; diff --git a/tests/macros-test/src/main.rs b/tests/macros-test/src/main.rs index 475151bd..81362d30 100644 --- a/tests/macros-test/src/main.rs +++ b/tests/macros-test/src/main.rs @@ -3049,7 +3049,7 @@ mod test { .map(|i| Ok(Tuple::new(None, vec![ DataValue::Int32(i), DataValue::Int32(i), - ])))) as Box>>) + ])))) as Box> + Send>) })); #[test] diff --git a/tests/sqllogictest/Cargo.toml b/tests/sqllogictest/Cargo.toml index 1c22a89d..031cce47 100644 --- a/tests/sqllogictest/Cargo.toml +++ b/tests/sqllogictest/Cargo.toml @@ -4,9 +4,24 @@ version = "0.4.0" edition = "2021" default-run = "sqllogictest-test" +[features] +default = ["lmdb"] +lmdb = ["kite_sql/lmdb"] +# Runs the suite through KiteSQL's async path, over SlateDB on an in-memory object store. +async = ["kite_sql/slatedb", "dep:async-trait", "dep:slatedb", "dep:tokio"] + +[[bin]] +name = "sqlite" +path = "src/bin/sqlite.rs" +required-features = ["lmdb"] + [dependencies] +async-trait = { version = "0.1", optional = true } clap = { version = "4" } -"kite_sql" = { path = "../..", default-features = false, features = ["copy", "decimal", "lmdb", "macros", "orm", "parser", "spill", "time"] } +"kite_sql" = { path = "../..", default-features = false, features = ["copy", "decimal", "macros", "orm", "parser", "spill", "time"] } +kite_sql_serde_macros = { path = "../../kite_sql_serde_macros" } glob = { version = "0.3" } sqllogictest = { version = "0.29.1" } +slatedb = { version = "0.18", optional = true, default-features = false } tempfile = { version = "3.10" } +tokio = { version = "1", features = ["rt-multi-thread"], optional = true } diff --git a/tests/sqllogictest/src/lib.rs b/tests/sqllogictest/src/lib.rs index 852b3130..fab8c1d1 100644 --- a/tests/sqllogictest/src/lib.rs +++ b/tests/sqllogictest/src/lib.rs @@ -12,18 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -use kite_sql::db::Database; +use kite_sql::db::{ConsumeMut, Database, TransactionIter}; use kite_sql::errors::DatabaseError; -use kite_sql::storage::lmdb::LmdbStorage; +use kite_sql::planner::PlanArena; +use kite_sql::storage::Storage; use kite_sql::types::value::DataValue; -use sqllogictest::{DBOutput, DefaultColumnType, DB}; +use kite_sql_serde_macros::maybe_async; +use sqllogictest::{DBOutput, DefaultColumnType}; use std::time::Instant; -pub struct SQLBase { - pub db: Database, +pub struct SQLBase { + pub db: Database, } -impl DB for SQLBase { +#[cfg(not(feature = "async"))] +impl sqllogictest::DB for SQLBase { type Error = DatabaseError; type ColumnType = DefaultColumnType; @@ -36,24 +39,57 @@ impl DB for SQLBase { } } +#[cfg(feature = "async")] +#[async_trait::async_trait] +impl sqllogictest::AsyncDB for SQLBase { + type Error = DatabaseError; + type ColumnType = DefaultColumnType; + + async fn run(&mut self, sql: &str) -> Result, Self::Error> { + let start = Instant::now(); + println!("|— Input SQL: {}", sql); + let output = run_sql(&mut self.db, sql, |_, value| value.to_string()).await?; + println!(" |— time spent: {:?}", start.elapsed()); + Ok(output) + } + + async fn shutdown(&mut self) {} +} + /// Runs every statement in `sql` and returns the output of the last one, rendering each value of /// column `i` with `format(i, value)`. -pub fn run_sql( - db: &mut Database, +#[maybe_async] +pub async fn run_sql( + db: &mut Database, sql: &str, - mut format: impl FnMut(usize, &DataValue) -> String, + format: impl FnMut(usize, &DataValue) -> String + Send, ) -> Result, DatabaseError> { - db.run_mut(sql, |iter| { + db.run_mut(sql, Render(format)).await +} + +struct Render(F); + +#[maybe_async] +impl String + Send> ConsumeMut for Render { + type Output = DBOutput; + + async fn consume<'a>( + mut self, + iter: &mut TransactionIter<'a, 'a, S::TransactionType<'a>, PlanArena<'a>>, + ) -> Result { let width = iter.schema(|schema| schema.len()); let types = vec![DefaultColumnType::Any; width]; let mut rows = Vec::new(); - while let Some(row) = iter.next_tuple(|_, tuple| { - tuple - .iter() - .enumerate() - .map(|(i, value)| format(i, value)) - .collect() - })? { + while let Some(row) = iter + .next_tuple(|_, tuple| { + tuple + .iter() + .enumerate() + .map(|(i, value)| (self.0)(i, value)) + .collect() + }) + .await? + { rows.push(row); } if rows.is_empty() { @@ -61,5 +97,5 @@ pub fn run_sql( } else { Ok(DBOutput::Rows { types, rows }) } - }) + } } diff --git a/tests/sqllogictest/src/main.rs b/tests/sqllogictest/src/main.rs index e7939d65..fea95b8c 100644 --- a/tests/sqllogictest/src/main.rs +++ b/tests/sqllogictest/src/main.rs @@ -21,6 +21,7 @@ use std::io; use std::io::Write; use std::path::Path; use std::time::Instant; +#[cfg(not(feature = "async"))] use tempfile::TempDir; #[derive(Parser, Debug)] @@ -42,11 +43,27 @@ fn main() { let mut file_num = 0; let start = Instant::now(); + #[cfg(not(feature = "async"))] let temp_dir = TempDir::new().expect("unable to create temporary working directory"); + #[cfg(not(feature = "async"))] let mut tester = Runner::new(|| { let db = DataBaseBuilder::path(temp_dir.path()).build_lmdb(); std::future::ready(db.map(|db| SQLBase { db })) }); + // The async path runs on SlateDB over an in-memory object store, whose background work needs + // a tokio runtime. + #[cfg(feature = "async")] + let runtime = tokio::runtime::Runtime::new().expect("failed to start tokio"); + #[cfg(feature = "async")] + let _runtime = runtime.enter(); + #[cfg(feature = "async")] + let mut tester = { + let storage = open_slatedb(&runtime); + Runner::new(move || { + let db = DataBaseBuilder::path(".").build_with_storage(storage.clone()); + std::future::ready(db.map(|db| SQLBase { db })) + }) + }; for slt_file in glob::glob(&args.path).expect("failed to find slt files") { let filepath = slt_file.expect("failed to read slt file"); @@ -71,6 +88,26 @@ fn main() { } } +#[cfg(feature = "async")] +fn open_slatedb(runtime: &tokio::runtime::Runtime) -> kite_sql::storage::slatedb::SlateDbStorage { + use slatedb::object_store::memory::InMemory; + use std::sync::Arc; + use std::time::Duration; + + let settings = slatedb::Settings { + flush_interval: Some(Duration::from_millis(1)), + ..Default::default() + }; + let db = runtime + .block_on( + slatedb::Db::builder("kitesql-slt", Arc::new(InMemory::new())) + .with_settings(settings) + .build(), + ) + .expect("failed to open slatedb"); + kite_sql::storage::slatedb::SlateDbStorage::new(Arc::new(db)) +} + fn init_20000_row_csv() -> io::Result<()> { let path = "tests/data/row_20000.csv"; diff --git a/tpcc/Cargo.toml b/tpcc/Cargo.toml index 82cc9969..6a974f1a 100644 --- a/tpcc/Cargo.toml +++ b/tpcc/Cargo.toml @@ -4,17 +4,25 @@ version = "0.1.1" edition = "2021" [features] -pprof = ["dep:pprof"] +default = ["kitesql-sync"] +# The sync storages: KiteSQL on LMDB and RocksDB, and the dual check against SQLite. +kitesql-sync = ["kite_sql/lmdb", "kite_sql/rocksdb"] +# KiteSQL's async path on SlateDB over S3; excludes `kitesql-sync`. Build it with +# `--no-default-features --features slatedb-s3`. +slatedb-s3 = ["kite_sql/slatedb-s3"] +pprof = ["dep:pprof"] [dependencies] clap = { version = "4", features = ["derive"] } chrono = { version = "0.4" } hdrhistogram = { version = "7", default-features = false } -kite_sql = { path = "..", package = "kite_sql", features = ["rocksdb", "lmdb", "decimal"] } +kite_sql = { path = "..", package = "kite_sql", default-features = false, features = ["decimal", "macros", "parser", "time"] } +futures = { version = "0.3" } indicatif = { version = "0.17" } rand = { version = "0.8" } rust_decimal = { version = "1" } sqlite = { version = "0.34" } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } [target.'cfg(unix)'.dependencies] pprof = { version = "0.15", features = ["flamegraph"], optional = true } diff --git a/tpcc/README.md b/tpcc/README.md index d9a7d336..c04d8015 100644 --- a/tpcc/README.md +++ b/tpcc/README.md @@ -16,15 +16,37 @@ Common options: | Option | Default | Meaning | | --- | --- | --- | -| `--backend` | `kitesql-lmdb` | `kitesql-lmdb`, `kitesql-rocksdb`, `kitesql-optimistic-rocksdb`, `sqlite`, or `dual` | +| `--backend` | `kitesql-lmdb` | `kitesql-lmdb`, `kitesql-rocksdb`, `kitesql-optimistic-rocksdb`, `sqlite`, or `dual`; `kitesql-slatedb-s3` or `sqlite` in the [SlateDB build](#slatedb-on-s3) | | `--path` | `kite_sql_tpcc` | Database path | -| `--threads` | `1` | Concurrent terminals, each on its own connection | +| `--threads` | `1` | Threads that run the terminals | +| `--terminals` | `--threads` | Concurrent terminals; terminal `i` has home warehouse `i % num_ware + 1`. More terminals than threads needs an async storage (SlateDB) | | `--num-ware` | `1` | Number of warehouses | | `--measure-time` | `720` | Measured seconds after loading | -| `--max-retry` | `5` | Retries per failed transaction | -| `--sync` | off | Fsync on commit (SQLite: `synchronous = FULL` instead of `OFF`) | +| `--max-retry` | `5` | Retries per transaction; conflicts are counted apart from failures and retried after a random backoff | +| `--sync` | off | Wait for each commit to be durable: fsync for LMDB and RocksDB, `synchronous = FULL` instead of `OFF` for SQLite, the write to S3 for SlateDB | | `--rocksdb-stats` | off | Print RocksDB statistics | +## SlateDB on S3 +KiteSQL on SlateDB runs on the async path, which excludes the sync storages, so it is a separate build: + +```shell +AWS_REGION=us-east-1 AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... \ +cargo run -p tpcc --release --no-default-features --features slatedb-s3 -- \ + --s3-bucket [--s3-endpoint http://127.0.0.1:9000 --s3-allow-http] +``` + +| Option | Default | Meaning | +| --- | --- | --- | +| `--s3-bucket` | required | Bucket; credentials and region come from `AWS_*` or the default AWS chain | +| `--s3-endpoint` | AWS | Endpoint of an S3-compatible store | +| `--s3-allow-http` | off | Allow a plain `http://` endpoint | +| `--path` | `kite_sql_tpcc` | Key prefix; each run writes under a new `/` | +| `--slatedb-memory-cache-mb` | `4096` | Memory of the block cache | +| `--slatedb-disk-cache` | `kite_sql_tpcc_cache` | Local disk tier of the block cache, cleared before each run | +| `--slatedb-disk-cache-mb` | `65536` | Its capacity; the files grow as they fill. `0` turns it off | + +Blocks written by compactions also go into the cache. Without a cache that holds the data, every miss costs an S3 request in the transaction, and TpmC drops by orders of magnitude. + ## Stable Performance Runner Use the Python runner for local performance comparisons: @@ -34,7 +56,7 @@ Use the Python runner for local performance comparisons: It runs `kitesql-lmdb`, `kitesql-rocksdb`, and `sqlite` (select a subset with `--variants`), pins TPCC to one CPU, and waits before each variant until the machine has enough consecutive stable samples. -TPCC runs as a single worker, so its speed depends on the core it lands on. On hybrid CPUs (P-cores and E-cores) an unpinned run can differ by 30-60% depending on scheduler placement. The runner therefore pins every variant, including its background threads, to the strongest core by default: the one with the highest `cpuinfo_max_freq`, lowest id on ties, skipping the physical core of `cpu0`. On hybrid CPUs this is a P-core, and on CPUs with Turbo Boost Max 3.0 it is one of the favored P-cores. Use `--cpu ` to choose one or `--no-cpu-pin` to disable pinning. The chosen CPU is recorded in `summary.md`. +By default TPCC runs as a single worker, so its speed depends on the core it lands on. On hybrid CPUs (P-cores and E-cores) an unpinned run can differ by 30-60% depending on scheduler placement. The runner therefore pins every variant, including its background threads, to the strongest core by default: the one with the highest `cpuinfo_max_freq`, lowest id on ties, skipping the physical core of `cpu0`. On hybrid CPUs this is a P-core, and on CPUs with Turbo Boost Max 3.0 it is one of the favored P-cores. Use `--cpu ` to choose one or `--no-cpu-pin` to disable pinning. The chosen CPU is recorded in `summary.md`. The runner waits until: @@ -65,21 +87,32 @@ The benchmark stores `history.h_date` as `timestamp(6)`, so high-throughput `Pay - i9-13900HX - 32.0 GB - KIOXIA-EXCERIA PLUS G3 SSD -- Tips: TPC-C currently runs as a single worker. ## 720s comparison -Local 720-second comparison on the machine above: +Local 720-second comparison on the machine above. Latencies are p90 in µs. + +1 thread, 1 warehouse: | Backend | TpmC | New-Order p90 (µs) | Payment p90 (µs) | Order-Status p90 (µs) | Delivery p90 (µs) | Stock-Level p90 (µs) | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| KiteSQL LMDB | 225720 | 240 | 44 | 25 | 326 | 179 | -| KiteSQL RocksDB | 48746 | 343 | 78 | 140 | 14351 | 587 | -| SQLite | 192110 | 270 | 40 | 33 | 342 | 231 | - -- Run date: `2026-11-05`. Latency is measured in microseconds and includes commit. -- Every row is pinned to `cpu8`, the highest-frequency P-core on this machine (5.4 GHz). -- All rows use `--num-ware 1`, `--max-retry 5`, and TPCC's default 720-second measure time. -- All rows run without `--sync`, so commits are not fsynced; SQLite uses WAL with `synchronous = OFF`. +| KiteSQL LMDB | 254023 | 211 | 38 | 25 | 293 | 175 | +| KiteSQL RocksDB | 47488 | 372 | 85 | 415 | 15471 | 389 | +| KiteSQL SlateDB (S3) | 12016 | 677 | 240 | 399 | 74047 | 1638 | +| SQLite | 214001 | 251 | 37 | 32 | 338 | 227 | + +4 threads, 4 warehouses: + +| Backend | TpmC | New-Order p90 (µs) | Payment p90 (µs) | Order-Status p90 (µs) | Delivery p90 (µs) | Stock-Level p90 (µs) | Conflicts | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| KiteSQL LMDB | 240584 | 930 | 780 | 29 | 890 | 207 | 0 | +| KiteSQL RocksDB | 212936 | 505 | 109 | 285 | 8991 | 400 | 529 | +| KiteSQL SlateDB (S3) | 19010 | 1749 | 1816 | 2481 | 66495 | 2523 | 51 | +| SQLite | 205896 | 254 | 41 | 45 | 336 | 380 | 0 | + +- Run date: `2026-11-09`. Latency is measured in microseconds and includes commit and retries. +- The 1-thread rows are pinned to `cpu8`, the highest-frequency P-core on this machine (5.4 GHz), background threads included. The 4-thread rows are pinned to `cpu8,10,2,4`, four distinct physical P-cores. These runs used `taskset` directly, since the stable runner covers neither threads nor SlateDB. +- The 4-thread rows give each terminal its own home warehouse, so they scale data with threads. +- All rows use `--max-retry 5` and run without `--sync`, so commits are not fsynced and SlateDB does not wait for S3; SQLite uses WAL with `synchronous = OFF`. The runner writes `summary.md` and per-variant logs to `tpcc/results//`. diff --git a/tpcc/src/backend/dual.rs b/tpcc/src/backend/dual.rs index df89d1a5..e986e431 100644 --- a/tpcc/src/backend/dual.rs +++ b/tpcc/src/backend/dual.rs @@ -12,18 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::kitesql_rocksdb::{KiteSqlRocksDbBackend, KiteSqlRocksDbTransaction, KiteSqlTxnResult}; +use super::kitesql::{KiteSqlBackend, KiteSqlTransaction, KiteSqlTxnResult}; use super::sqlite::{SqliteBackend, SqlitePreparedStatement, SqliteResult, SqliteTransaction}; use super::{ BackendControl, BackendTransaction, DbParam, KiteSqlPreparedStatement, PreparedStatement, - SimpleExecutor, StatementSpec, + RowVisitor, SimpleExecutor, StatementSpec, }; use crate::TpccError; +use kite_sql::storage::rocksdb::RocksStorage; use kite_sql::types::tuple::TupleImpl; use std::borrow::Cow; pub struct DualBackend { - kitesql: KiteSqlRocksDbBackend, + kitesql: KiteSqlBackend, sqlite: SqliteBackend, } @@ -42,7 +43,7 @@ impl PreparedStatement for DualPreparedStatement<'_> { impl DualBackend { pub fn new(path: &str, rocksdb_stats: bool) -> Result { Ok(Self { - kitesql: KiteSqlRocksDbBackend::new(path, rocksdb_stats, false)?, + kitesql: KiteSqlBackend::rocksdb(path, rocksdb_stats, false)?, sqlite: SqliteBackend::new_memory()?, }) } @@ -106,68 +107,73 @@ impl BackendControl for DualBackend { } impl SimpleExecutor for DualBackend { - fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError> { - self.kitesql.execute_batch(sql)?; + async fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError> { + self.kitesql.execute_batch(sql).await?; if let Some(stmt) = normalize_sqlite_sql(sql) { - self.sqlite.execute_batch(&stmt)?; + self.sqlite.execute_batch(&stmt).await?; } Ok(()) } } pub struct DualTransaction<'a> { - kitesql: KiteSqlRocksDbTransaction<'a>, + kitesql: KiteSqlTransaction<'a, RocksStorage>, sqlite: SqliteTransaction<'a>, } impl<'a> BackendTransaction for DualTransaction<'a> { type PreparedStatement = DualPreparedStatement<'a>; - fn execute_drain( + async fn execute_drain( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], + params: &[DbParam<'_>], ) -> Result<(), TpccError> { let spec = statement.spec.clone(); - let kitesql_iter = self.kitesql.execute_raw(&mut statement.kitesql, params)?; + let kitesql_iter = self + .kitesql + .execute_raw(&mut statement.kitesql, params) + .await?; let sqlite_iter = self.sqlite.execute_raw(&mut statement.sqlite, params)?; if is_select_sql(&spec) { - drain_and_compare_ordered(kitesql_iter, sqlite_iter, spec.sql) + drain_and_compare_ordered(kitesql_iter, sqlite_iter, spec.sql).await } else { drain_sqlite_iter(sqlite_iter)?; - drain_kitesql_iter(kitesql_iter) + drain_kitesql_iter(kitesql_iter).await } } - fn with_query_one( + async fn with_query_one( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, + params: &[DbParam<'_>], + visitor: RowVisitor<'_>, ) -> Result<(), TpccError> { - self.with_query_nth(statement, params, 0, visitor) + self.with_query_nth(statement, params, 0, visitor).await } - fn with_query_all( + async fn with_query_all( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, + params: &[DbParam<'_>], + visitor: RowVisitor<'_>, ) -> Result<(), TpccError> { let mut rows = Vec::new(); self.kitesql .with_query_all(&mut statement.kitesql, params, &mut |tuple| { rows.push(tuple.clone()); Ok(()) - })?; + }) + .await?; let mut sqlite_rows = Vec::new(); self.sqlite .with_query_all(&mut statement.sqlite, params, &mut |tuple| { sqlite_rows.push(tuple.clone()); Ok(()) - })?; + }) + .await?; // SQL without ORDER BY may return the same rows in different orders. for row in &rows { let Some(index) = sqlite_rows.iter().position(|values| row.values_eq(values)) else { @@ -190,29 +196,32 @@ impl<'a> BackendTransaction for DualTransaction<'a> { Ok(()) } - fn with_query_nth( + async fn with_query_nth( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], + params: &[DbParam<'_>], n: usize, - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, + visitor: RowVisitor<'_>, ) -> Result<(), TpccError> { let spec = statement.spec.clone(); - let kitesql_iter = self.kitesql.execute_raw(&mut statement.kitesql, params)?; + let kitesql_iter = self + .kitesql + .execute_raw(&mut statement.kitesql, params) + .await?; let sqlite_iter = self.sqlite.execute_raw(&mut statement.sqlite, params)?; if !is_select_sql(&spec) { drain_sqlite_iter(sqlite_iter)?; - return with_kitesql_nth(kitesql_iter, n, visitor); + return with_kitesql_nth(kitesql_iter, n, visitor).await; } - with_ordered_nth(kitesql_iter, sqlite_iter, spec.sql, n, visitor) + with_ordered_nth(kitesql_iter, sqlite_iter, spec.sql, n, visitor).await } - fn commit(self) -> Result<(), TpccError> { - self.sqlite.commit()?; - self.kitesql.commit() + async fn commit(self) -> Result<(), TpccError> { + self.sqlite.commit().await?; + self.kitesql.commit().await } } @@ -233,28 +242,29 @@ fn drain_sqlite_iter(mut iter: SqliteResult<'_, '_>) -> Result<(), TpccError> { Ok(()) } -fn drain_kitesql_iter( +async fn drain_kitesql_iter( mut iter: KiteSqlTxnResult<'_, '_, T>, ) -> Result<(), TpccError> { - while iter.skip_next_tuple()? {} + while iter.skip_next_tuple().await? {} Ok(()) } -fn with_kitesql_nth( +async fn with_kitesql_nth( mut iter: KiteSqlTxnResult<'_, '_, T>, n: usize, visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, ) -> Result<(), TpccError> { for _ in 0..n { - if !iter.skip_next_tuple()? { + if !iter.skip_next_tuple().await? { return Err(TpccError::EmptyTuples); } } - iter.with_next_tuple(|tuple| visitor(tuple))? + iter.with_next_tuple(|tuple| visitor(tuple)) + .await? .ok_or(TpccError::EmptyTuples) } -fn with_ordered_nth( +async fn with_ordered_nth( mut kitesql_iter: KiteSqlTxnResult<'_, '_, T>, mut sqlite_iter: SqliteResult<'_, '_>, sql: &'static str, @@ -265,35 +275,38 @@ fn with_ordered_nth( let mut index = 0usize; loop { - match kitesql_iter.with_next_tuple(|kitesql_tuple| { - let sqlite_tuple = match sqlite_iter.next() { - Some(row) => row?, - None => { + match kitesql_iter + .with_next_tuple(|kitesql_tuple| { + let sqlite_tuple = match sqlite_iter.next() { + Some(row) => row?, + None => { + return Err(TpccError::BackendMismatch(format!( + "SQLite returned fewer rows for SQL: {}", + sql + ))) + } + }; + if !kitesql_tuple.iter().eq(&sqlite_tuple.values) { + println!("[Dual] mismatch SQL: {}", sql); + println!( + " KiteSQL row: {:?}", + kitesql_tuple.iter().collect::>() + ); + println!(" SQLite row: {:?}", sqlite_tuple.values); return Err(TpccError::BackendMismatch(format!( - "SQLite returned fewer rows for SQL: {}", + "Result mismatch for SQL: {}", sql - ))) + ))); } - }; - if !kitesql_tuple.iter().eq(&sqlite_tuple.values) { - println!("[Dual] mismatch SQL: {}", sql); - println!( - " KiteSQL row: {:?}", - kitesql_tuple.iter().collect::>() - ); - println!(" SQLite row: {:?}", sqlite_tuple.values); - return Err(TpccError::BackendMismatch(format!( - "Result mismatch for SQL: {}", - sql - ))); - } - if index == n { - visitor(&sqlite_tuple.into())?; - visited = true; - } - index += 1; - Ok(()) - })? { + if index == n { + visitor(&sqlite_tuple.into())?; + visited = true; + } + index += 1; + Ok(()) + }) + .await? + { Some(()) => {} None => { if let Some(extra) = sqlite_iter.next() { @@ -313,36 +326,39 @@ fn with_ordered_nth( } } -fn drain_and_compare_ordered( +async fn drain_and_compare_ordered( mut kitesql_iter: KiteSqlTxnResult<'_, '_, T>, mut sqlite_iter: SqliteResult<'_, '_>, sql: &'static str, ) -> Result<(), TpccError> { loop { - match kitesql_iter.with_next_tuple(|kitesql_tuple| { - let sqlite_tuple = match sqlite_iter.next() { - Some(row) => row?, - None => { + match kitesql_iter + .with_next_tuple(|kitesql_tuple| { + let sqlite_tuple = match sqlite_iter.next() { + Some(row) => row?, + None => { + return Err(TpccError::BackendMismatch(format!( + "SQLite returned fewer rows for SQL: {}", + sql + ))) + } + }; + if !kitesql_tuple.iter().eq(&sqlite_tuple.values) { + println!("[Dual] mismatch SQL: {}", sql); + println!( + " KiteSQL row: {:?}", + kitesql_tuple.iter().collect::>() + ); + println!(" SQLite row: {:?}", sqlite_tuple.values); return Err(TpccError::BackendMismatch(format!( - "SQLite returned fewer rows for SQL: {}", + "Result mismatch for SQL: {}", sql - ))) + ))); } - }; - if !kitesql_tuple.iter().eq(&sqlite_tuple.values) { - println!("[Dual] mismatch SQL: {}", sql); - println!( - " KiteSQL row: {:?}", - kitesql_tuple.iter().collect::>() - ); - println!(" SQLite row: {:?}", sqlite_tuple.values); - return Err(TpccError::BackendMismatch(format!( - "Result mismatch for SQL: {}", - sql - ))); - } - Ok(()) - })? { + Ok(()) + }) + .await? + { Some(()) => {} None => { if let Some(extra) = sqlite_iter.next() { diff --git a/tpcc/src/backend/kitesql.rs b/tpcc/src/backend/kitesql.rs new file mode 100644 index 00000000..1891e1d7 --- /dev/null +++ b/tpcc/src/backend/kitesql.rs @@ -0,0 +1,320 @@ +// Copyright 2024 KipData/KiteSQL +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! KiteSQL on any of its storages: LMDB and RocksDB in the default build, SlateDB on S3 in the +//! `slatedb-s3` build, where KiteSQL's API is async. + +use super::{ + BackendControl, BackendTransaction, DbParam, KiteSqlPreparedStatement, RowVisitor, + SimpleExecutor, StatementSpec, +}; +use crate::TpccError; +use kite_sql::binder::{command_type, CommandType}; +use kite_sql::db::{DataBaseBuilder, Database, ProgramIter, Statement, Transaction}; +use kite_sql::storage::{Storage, StorageTransaction}; +use kite_sql::types::tuple::TupleImpl; + +/// A call into KiteSQL, awaited in the `slatedb-s3` build, where KiteSQL's API is async. +#[cfg(not(feature = "slatedb-s3"))] +macro_rules! kite { + ($call:expr) => { + $call + }; +} + +#[cfg(feature = "slatedb-s3")] +macro_rules! kite { + ($call:expr) => { + $call.await + }; +} + +/// `Send` in the `slatedb-s3` build, where KiteSQL runs what reads a row inside a `Send` future. +#[cfg(feature = "slatedb-s3")] +pub trait MaybeSend: Send {} +#[cfg(feature = "slatedb-s3")] +impl MaybeSend for T {} +#[cfg(not(feature = "slatedb-s3"))] +pub trait MaybeSend {} +#[cfg(not(feature = "slatedb-s3"))] +impl MaybeSend for T {} + +pub struct KiteSqlBackend { + database: Database, +} + +#[cfg(feature = "kitesql-sync")] +impl KiteSqlBackend { + pub fn lmdb(path: &str, sync: bool) -> Result { + Ok(Self { + database: DataBaseBuilder::path(path).sync(sync).build_lmdb()?, + }) + } +} + +#[cfg(feature = "kitesql-sync")] +impl KiteSqlBackend { + pub fn rocksdb(path: &str, rocksdb_stats: bool, sync: bool) -> Result { + Ok(Self { + database: DataBaseBuilder::path(path) + .storage_statistics(rocksdb_stats) + .sync(sync) + .build_rocksdb()?, + }) + } +} + +#[cfg(feature = "kitesql-sync")] +impl KiteSqlBackend { + pub fn optimistic_rocksdb( + path: &str, + rocksdb_stats: bool, + sync: bool, + ) -> Result { + Ok(Self { + database: DataBaseBuilder::path(path) + .storage_statistics(rocksdb_stats) + .sync(sync) + .build_optimistic()?, + }) + } +} + +#[cfg(feature = "slatedb-s3")] +impl KiteSqlBackend { + /// KiteSQL on a storage opened by the caller, such as SlateDB, which the caller closes. + pub fn with_storage(storage: S) -> Result { + Ok(Self { + database: DataBaseBuilder::path(".").build_with_storage(storage)?, + }) + } +} + +impl BackendControl for KiteSqlBackend { + type PreparedStatement<'a> + = KiteSqlPreparedStatement<'a> + where + Self: 'a; + + type Transaction<'a> + = KiteSqlTransaction<'a, S> + where + Self: 'a; + + fn prepare_statements( + &self, + specs: &[Vec], + ) -> Result>>, TpccError> { + let mut groups = Vec::with_capacity(specs.len()); + for group in specs { + let mut prepared = Vec::with_capacity(group.len()); + for spec in group { + prepared.push(KiteSqlPreparedStatement { + program: self.database.program_sql(spec.sql, &spec.parameters)?, + spec: spec.clone(), + }); + } + groups.push(prepared); + } + Ok(groups) + } + + type Worker<'a> + = &'a Self + where + Self: 'a; + + fn new_transaction(&self, read_only: bool) -> Result, TpccError> { + Ok(KiteSqlTransaction { + inner: self.database.new_transaction(read_only)?, + }) + } + + fn worker(&self) -> Result, TpccError> { + Ok(self) + } + + fn storage_metrics(&self) -> Option { + self.database + .storage_metrics() + .map(|metrics| metrics.to_string()) + } + + fn is_async(&self) -> bool { + cfg!(feature = "slatedb-s3") + } +} + +impl SimpleExecutor for KiteSqlBackend { + async fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError> { + let database = &mut self.database; + match KiteSqlBatch::classify(&database.prepare_all(sql)?)? { + KiteSqlBatch::Ddl => kite!(database.ddl(sql))?, + KiteSqlBatch::Analyze(tables) => { + for table_name in tables { + let start = std::time::Instant::now(); + println!("[KiteSQL Analyze: {table_name}] started"); + kite!(database.analyze(&table_name))?; + println!( + "[KiteSQL Analyze: {table_name}] completed in {:.2}s", + start.elapsed().as_secs_f64() + ); + } + } + KiteSqlBatch::Statements => { + let mut transaction = database.new_transaction(false)?; + let iter = kite!(transaction.run(sql))?; + kite!(iter.done())?; + kite!(transaction.commit())?; + } + } + Ok(()) + } +} + +enum KiteSqlBatch { + Ddl, + Analyze(Vec), + Statements, +} + +impl KiteSqlBatch { + fn classify(statements: &[Statement]) -> Result { + let all_ddl = statements.iter().try_fold(true, |all_ddl, statement| { + Ok::<_, kite_sql::errors::DatabaseError>( + all_ddl && matches!(command_type(statement)?, CommandType::DDL), + ) + })?; + if all_ddl { + return Ok(Self::Ddl); + } + if !statements + .iter() + .all(|statement| matches!(statement, Statement::Analyze(_))) + { + return Ok(Self::Statements); + } + let mut tables = Vec::with_capacity(statements.len()); + for statement in statements { + let Statement::Analyze(analyze) = statement else { + unreachable!("checked above") + }; + let table_name = analyze.table_name.as_ref().ok_or_else(|| { + kite_sql::errors::DatabaseError::UnsupportedStmt( + "ANALYZE requires table name".to_string(), + ) + })?; + tables.push(table_name.to_string()); + } + Ok(Self::Analyze(tables)) + } +} + +pub struct KiteSqlTransaction<'a, S: Storage> { + inner: Transaction<'a, S>, +} + +impl<'a, S: Storage> KiteSqlTransaction<'a, S> { + pub(crate) async fn execute_raw<'b>( + &'b mut self, + statement: &'b mut KiteSqlPreparedStatement<'a>, + params: &[DbParam<'_>], + ) -> Result>, TpccError> { + Ok(KiteSqlTxnResult(kite!(statement + .program + .execute(&mut self.inner, params))?)) + } +} + +impl<'a, S: Storage> BackendTransaction for KiteSqlTransaction<'a, S> { + type PreparedStatement = KiteSqlPreparedStatement<'a>; + + async fn execute_drain( + &mut self, + statement: &mut Self::PreparedStatement, + params: &[DbParam<'_>], + ) -> Result<(), TpccError> { + let mut iter = self.execute_raw(statement, params).await?; + while iter.skip_next_tuple().await? {} + Ok(()) + } + + async fn with_query_one( + &mut self, + statement: &mut Self::PreparedStatement, + params: &[DbParam<'_>], + visitor: RowVisitor<'_>, + ) -> Result<(), TpccError> { + let mut iter = self.execute_raw(statement, params).await?; + iter.with_next_tuple(|tuple| visitor(tuple)) + .await? + .ok_or(TpccError::EmptyTuples) + } + + async fn with_query_all( + &mut self, + statement: &mut Self::PreparedStatement, + params: &[DbParam<'_>], + visitor: RowVisitor<'_>, + ) -> Result<(), TpccError> { + let mut iter = self.execute_raw(statement, params).await?; + while iter + .with_next_tuple(|tuple| visitor(tuple)) + .await? + .is_some() + {} + Ok(()) + } + + async fn with_query_nth( + &mut self, + statement: &mut Self::PreparedStatement, + params: &[DbParam<'_>], + n: usize, + visitor: RowVisitor<'_>, + ) -> Result<(), TpccError> { + let mut iter = self.execute_raw(statement, params).await?; + for _ in 0..n { + if !iter.skip_next_tuple().await? { + return Err(TpccError::EmptyTuples); + } + } + iter.with_next_tuple(|tuple| visitor(tuple)) + .await? + .ok_or(TpccError::EmptyTuples) + } + + async fn commit(self) -> Result<(), TpccError> { + kite!(self.inner.commit())?; + Ok(()) + } +} + +pub struct KiteSqlTxnResult<'b, 'a, T: StorageTransaction + 'a>(ProgramIter<'b, 'a, T>); + +impl<'a, T: StorageTransaction + 'a> KiteSqlTxnResult<'_, 'a, T> { + pub(crate) async fn skip_next_tuple(&mut self) -> Result { + Ok(kite!(self.0.next_tuple(|_, _| ()))?.is_some()) + } + + pub(crate) async fn with_next_tuple( + &mut self, + f: impl FnOnce(&TupleImpl) -> Result + MaybeSend, + ) -> Result, TpccError> { + match kite!(self.0.next_tuple(|_, tuple| f(tuple)))? { + Some(result) => result.map(Some), + None => Ok(None), + } + } +} diff --git a/tpcc/src/backend/kitesql_lmdb.rs b/tpcc/src/backend/kitesql_lmdb.rs deleted file mode 100644 index b2b11cf4..00000000 --- a/tpcc/src/backend/kitesql_lmdb.rs +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2024 KipData/KiteSQL -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::kitesql_rocksdb::{execute_kitesql_batch, KiteSqlTxnResult}; -use super::{ - BackendControl, BackendTransaction, DbParam, KiteSqlPreparedStatement, SimpleExecutor, - StatementSpec, -}; -use crate::TpccError; -use kite_sql::db::{DataBaseBuilder, Database, Transaction}; -use kite_sql::storage::lmdb::{LmdbStorage, LmdbTransaction as KiteSqlLmdbTransaction}; -use kite_sql::types::tuple::TupleImpl; - -pub struct KiteSqlLmdbBackend { - database: Database, -} - -impl KiteSqlLmdbBackend { - pub fn new(path: &str, sync: bool) -> Result { - Ok(Self { - database: DataBaseBuilder::path(path).sync(sync).build_lmdb()?, - }) - } - - fn prepare_spec_groups( - &self, - specs: &[Vec], - ) -> Result>>, TpccError> { - let mut groups = Vec::with_capacity(specs.len()); - for group in specs { - let mut prepared = Vec::with_capacity(group.len()); - for spec in group { - prepared.push(KiteSqlPreparedStatement { - program: self.database.program_sql(spec.sql, &spec.parameters)?, - spec: spec.clone(), - }); - } - groups.push(prepared); - } - Ok(groups) - } - - fn start_transaction( - &self, - read_only: bool, - ) -> Result, TpccError> { - Ok(KiteSqlLmdbTransactionWrapper { - inner: self.database.new_transaction(read_only)?, - }) - } -} - -impl BackendControl for KiteSqlLmdbBackend { - type PreparedStatement<'a> - = KiteSqlPreparedStatement<'a> - where - Self: 'a; - - type Transaction<'a> - = KiteSqlLmdbTransactionWrapper<'a> - where - Self: 'a; - - fn prepare_statements( - &self, - specs: &[Vec], - ) -> Result>>, TpccError> { - self.prepare_spec_groups(specs) - } - - type Worker<'a> - = &'a Self - where - Self: 'a; - - fn new_transaction(&self, read_only: bool) -> Result, TpccError> { - self.start_transaction(read_only) - } - - fn worker(&self) -> Result, TpccError> { - Ok(self) - } -} - -impl SimpleExecutor for KiteSqlLmdbBackend { - fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError> { - execute_kitesql_batch(&mut self.database, sql) - } -} - -pub struct KiteSqlLmdbTransactionWrapper<'a> { - inner: Transaction<'a, LmdbStorage>, -} - -impl<'a> KiteSqlLmdbTransactionWrapper<'a> { - pub(crate) fn execute_raw<'b>( - &'b mut self, - statement: &'b mut KiteSqlPreparedStatement<'a>, - params: &[DbParam], - ) -> Result>, TpccError> { - Ok(KiteSqlTxnResult( - statement.program.execute(&mut self.inner, params)?, - )) - } -} - -impl<'a> BackendTransaction for KiteSqlLmdbTransactionWrapper<'a> { - type PreparedStatement = KiteSqlPreparedStatement<'a>; - - fn execute_drain( - &mut self, - statement: &mut Self::PreparedStatement, - params: &[DbParam], - ) -> Result<(), TpccError> { - let mut iter = self.execute_raw(statement, params)?; - while iter.skip_next_tuple()? {} - Ok(()) - } - - fn with_query_one( - &mut self, - statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, - ) -> Result<(), TpccError> { - let mut iter = self.execute_raw(statement, params)?; - iter.with_next_tuple(|tuple| visitor(tuple))? - .ok_or(TpccError::EmptyTuples) - } - - fn with_query_all( - &mut self, - statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, - ) -> Result<(), TpccError> { - let mut iter = self.execute_raw(statement, params)?; - while iter.with_next_tuple(|tuple| visitor(tuple))?.is_some() {} - Ok(()) - } - - fn with_query_nth( - &mut self, - statement: &mut Self::PreparedStatement, - params: &[DbParam], - n: usize, - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, - ) -> Result<(), TpccError> { - let mut iter = self.execute_raw(statement, params)?; - for _ in 0..n { - if !iter.skip_next_tuple()? { - return Err(TpccError::EmptyTuples); - } - } - iter.with_next_tuple(|tuple| visitor(tuple))? - .ok_or(TpccError::EmptyTuples) - } - - fn commit(self) -> Result<(), TpccError> { - self.inner.commit()?; - Ok(()) - } -} diff --git a/tpcc/src/backend/kitesql_rocksdb.rs b/tpcc/src/backend/kitesql_rocksdb.rs deleted file mode 100644 index 6dda6fc8..00000000 --- a/tpcc/src/backend/kitesql_rocksdb.rs +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2024 KipData/KiteSQL -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::{ - BackendControl, BackendTransaction, DbParam, KiteSqlPreparedStatement, SimpleExecutor, - StatementSpec, -}; -use crate::TpccError; -use kite_sql::binder::{command_type, CommandType}; -use kite_sql::db::{DataBaseBuilder, Database, ProgramIter, Statement, Transaction}; -use kite_sql::storage::rocksdb::{OptimisticRocksStorage, RocksStorage}; -use kite_sql::storage::{Storage, StorageTransaction}; -use kite_sql::types::tuple::TupleImpl; - -pub type KiteSqlRocksDbBackend = KiteSqlRocksBackend; -pub type KiteSqlOptimisticRocksDbBackend = KiteSqlRocksBackend; -pub type KiteSqlRocksDbTransaction<'a> = KiteSqlRocksTransaction<'a, RocksStorage>; - -pub struct KiteSqlRocksBackend { - database: Database, -} - -impl KiteSqlRocksDbBackend { - pub fn new(path: &str, rocksdb_stats: bool, sync: bool) -> Result { - Ok(Self { - database: DataBaseBuilder::path(path) - .storage_statistics(rocksdb_stats) - .sync(sync) - .build_rocksdb()?, - }) - } -} - -impl KiteSqlOptimisticRocksDbBackend { - pub fn new(path: &str, rocksdb_stats: bool, sync: bool) -> Result { - Ok(Self { - database: DataBaseBuilder::path(path) - .storage_statistics(rocksdb_stats) - .sync(sync) - .build_optimistic()?, - }) - } -} - -impl KiteSqlRocksBackend { - fn prepare_spec_groups( - &self, - specs: &[Vec], - ) -> Result>>, TpccError> { - let mut groups = Vec::with_capacity(specs.len()); - for group in specs { - let mut prepared = Vec::with_capacity(group.len()); - for spec in group { - prepared.push(KiteSqlPreparedStatement { - program: self.database.program_sql(spec.sql, &spec.parameters)?, - spec: spec.clone(), - }); - } - groups.push(prepared); - } - Ok(groups) - } - - fn start_transaction( - &self, - read_only: bool, - ) -> Result, TpccError> { - Ok(KiteSqlRocksTransaction { - inner: self.database.new_transaction(read_only)?, - }) - } -} - -impl BackendControl for KiteSqlRocksBackend { - type PreparedStatement<'a> - = KiteSqlPreparedStatement<'a> - where - Self: 'a; - - type Transaction<'a> - = KiteSqlRocksTransaction<'a, S> - where - Self: 'a; - - fn prepare_statements( - &self, - specs: &[Vec], - ) -> Result>>, TpccError> { - self.prepare_spec_groups(specs) - } - - type Worker<'a> - = &'a Self - where - Self: 'a; - - fn new_transaction(&self, read_only: bool) -> Result, TpccError> { - self.start_transaction(read_only) - } - - fn worker(&self) -> Result, TpccError> { - Ok(self) - } - - fn storage_metrics(&self) -> Option { - self.database - .storage_metrics() - .map(|metrics| metrics.to_string()) - } -} - -impl SimpleExecutor for KiteSqlRocksBackend { - fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError> { - execute_kitesql_batch(&mut self.database, sql) - } -} - -pub(crate) fn execute_kitesql_batch( - database: &mut Database, - sql: &str, -) -> Result<(), TpccError> { - let statements = database.prepare_all(sql)?; - let all_ddl = statements.iter().try_fold(true, |all_ddl, statement| { - Ok::<_, kite_sql::errors::DatabaseError>( - all_ddl && matches!(command_type(statement)?, CommandType::DDL), - ) - })?; - if all_ddl { - database.ddl(sql)?; - return Ok(()); - } - - if statements - .iter() - .all(|statement| matches!(statement, Statement::Analyze(_))) - { - for statement in statements { - let Statement::Analyze(analyze) = statement else { - unreachable!("checked above") - }; - let table_name = analyze.table_name.ok_or_else(|| { - kite_sql::errors::DatabaseError::UnsupportedStmt( - "ANALYZE requires table name".to_string(), - ) - })?; - let table_name = table_name.to_string(); - let start = std::time::Instant::now(); - println!("[KiteSQL Analyze: {table_name}] started"); - database.analyze(&table_name)?; - println!( - "[KiteSQL Analyze: {table_name}] completed in {:.2}s", - start.elapsed().as_secs_f64() - ); - } - return Ok(()); - } - - let mut transaction = database.new_transaction(false)?; - transaction.run(sql)?.done()?; - transaction.commit()?; - Ok(()) -} - -pub struct KiteSqlRocksTransaction<'a, S: Storage> { - inner: Transaction<'a, S>, -} - -impl<'a, S: Storage> KiteSqlRocksTransaction<'a, S> { - pub(crate) fn execute_raw<'b>( - &'b mut self, - statement: &'b mut KiteSqlPreparedStatement<'a>, - params: &[DbParam], - ) -> Result>, TpccError> { - Ok(KiteSqlTxnResult( - statement.program.execute(&mut self.inner, params)?, - )) - } -} - -impl<'a, S: Storage> BackendTransaction for KiteSqlRocksTransaction<'a, S> { - type PreparedStatement = KiteSqlPreparedStatement<'a>; - - fn execute_drain( - &mut self, - statement: &mut Self::PreparedStatement, - params: &[DbParam], - ) -> Result<(), TpccError> { - let mut iter = self.execute_raw(statement, params)?; - while iter.skip_next_tuple()? {} - Ok(()) - } - - fn with_query_one( - &mut self, - statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, - ) -> Result<(), TpccError> { - let mut iter = self.execute_raw(statement, params)?; - iter.with_next_tuple(|tuple| visitor(tuple))? - .ok_or(TpccError::EmptyTuples) - } - - fn with_query_all( - &mut self, - statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, - ) -> Result<(), TpccError> { - let mut iter = self.execute_raw(statement, params)?; - while iter.with_next_tuple(|tuple| visitor(tuple))?.is_some() {} - Ok(()) - } - - fn with_query_nth( - &mut self, - statement: &mut Self::PreparedStatement, - params: &[DbParam], - n: usize, - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, - ) -> Result<(), TpccError> { - let mut iter = self.execute_raw(statement, params)?; - for _ in 0..n { - if !iter.skip_next_tuple()? { - return Err(TpccError::EmptyTuples); - } - } - iter.with_next_tuple(|tuple| visitor(tuple))? - .ok_or(TpccError::EmptyTuples) - } - - fn commit(self) -> Result<(), TpccError> { - self.inner.commit()?; - Ok(()) - } -} - -pub struct KiteSqlTxnResult<'b, 'a, T: StorageTransaction + 'a>(pub(crate) ProgramIter<'b, 'a, T>); - -impl<'a, T: StorageTransaction + 'a> KiteSqlTxnResult<'_, 'a, T> { - pub(crate) fn skip_next_tuple(&mut self) -> Result { - Ok(self - .0 - .next_tuple(|_, _| ()) - .map_err(TpccError::from)? - .is_some()) - } - - pub(crate) fn with_next_tuple( - &mut self, - f: impl FnOnce(&TupleImpl) -> Result, - ) -> Result, TpccError> { - match self - .0 - .next_tuple(|_, tuple| f(tuple)) - .map_err(TpccError::from)? - { - Some(result) => result.map(Some), - None => Ok(None), - } - } -} diff --git a/tpcc/src/backend/mod.rs b/tpcc/src/backend/mod.rs index 58f4efc3..ac351d5d 100644 --- a/tpcc/src/backend/mod.rs +++ b/tpcc/src/backend/mod.rs @@ -13,9 +13,9 @@ use std::borrow::Borrow; // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(feature = "kitesql-sync")] pub mod dual; -pub mod kitesql_lmdb; -pub mod kitesql_rocksdb; +pub mod kitesql; pub mod sqlite; use crate::TpccError; @@ -25,8 +25,12 @@ use kite_sql::types::LogicalType; pub type DbParam<'a> = (usize, Param<'a>); +pub type RowVisitor<'v> = &'v mut (dyn FnMut(&TupleImpl) -> Result<(), TpccError> + Send); + +// Only the workload in this binary calls these, so the futures need no `Send` bound. +#[allow(async_fn_in_trait)] pub trait SimpleExecutor { - fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError>; + async fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError>; } pub trait BackendControl: SimpleExecutor { @@ -53,40 +57,47 @@ pub trait BackendControl: SimpleExecutor { fn storage_metrics(&self) -> Option { None } + + /// Whether transactions wait on asynchronous storage, so that several terminals on one + /// thread overlap their waits. Terminals of a sync storage run one after another instead. + fn is_async(&self) -> bool { + false + } } +#[allow(async_fn_in_trait)] pub trait BackendTransaction { type PreparedStatement: PreparedStatement; - fn execute_drain( + async fn execute_drain( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], + params: &[DbParam<'_>], ) -> Result<(), TpccError>; - fn with_query_one( + async fn with_query_one( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, + params: &[DbParam<'_>], + visitor: RowVisitor<'_>, ) -> Result<(), TpccError>; - fn with_query_all( + async fn with_query_all( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, + params: &[DbParam<'_>], + visitor: RowVisitor<'_>, ) -> Result<(), TpccError>; - fn with_query_nth( + async fn with_query_nth( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], + params: &[DbParam<'_>], n: usize, - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, + visitor: RowVisitor<'_>, ) -> Result<(), TpccError>; - fn commit(self) -> Result<(), TpccError>; + async fn commit(self) -> Result<(), TpccError>; } #[derive(Clone)] @@ -98,9 +109,11 @@ pub struct StatementSpec { pub struct KiteSqlPreparedStatement<'a> { pub program: Program<'a>, + #[cfg_attr(not(feature = "kitesql-sync"), allow(dead_code))] pub spec: StatementSpec, } +#[cfg_attr(not(feature = "kitesql-sync"), allow(dead_code))] pub trait PreparedStatement { fn spec(&self) -> &StatementSpec; } diff --git a/tpcc/src/backend/sqlite.rs b/tpcc/src/backend/sqlite.rs index e11b1cf8..714b490d 100644 --- a/tpcc/src/backend/sqlite.rs +++ b/tpcc/src/backend/sqlite.rs @@ -13,12 +13,13 @@ // limitations under the License. use super::{ - BackendControl, BackendTransaction, DbParam, PreparedStatement, SimpleExecutor, StatementSpec, + BackendControl, BackendTransaction, DbParam, PreparedStatement, RowVisitor, SimpleExecutor, + StatementSpec, }; use crate::TpccError; use chrono::{NaiveDateTime, TimeZone, Utc}; use kite_sql::db::Param; -use kite_sql::types::tuple::{Tuple, TupleImpl}; +use kite_sql::types::tuple::Tuple; use kite_sql::types::value::{DataValue, Utf8Type}; use kite_sql::types::{CharLengthUnits, LogicalType}; use rust_decimal::Decimal; @@ -37,6 +38,7 @@ impl SqliteBackend { Ok(Self { connection, path }) } + #[cfg_attr(not(feature = "kitesql-sync"), allow(dead_code))] pub fn new_memory() -> Result { let connection = Connection::open(":memory:")?; configure_sqlite(&connection, false)?; @@ -113,7 +115,7 @@ impl BackendControl for SqliteBackend { } impl SimpleExecutor for SqliteBackend { - fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError> { + async fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError> { if let Some(stmt) = normalize_sqlite_sql(sql) { self.connection.execute(&stmt)?; } @@ -141,7 +143,7 @@ impl<'a> SqliteTransaction<'a> { pub(crate) fn execute_raw<'b>( &'b mut self, statement: &'b mut SqlitePreparedStatement<'a>, - params: &[DbParam], + params: &[DbParam<'_>], ) -> Result, TpccError> { statement.statement.reset()?; bind_params(&mut statement.statement, params)?; @@ -160,10 +162,10 @@ impl Drop for SqliteTransaction<'_> { impl<'a> BackendTransaction for SqliteTransaction<'a> { type PreparedStatement = SqlitePreparedStatement<'a>; - fn execute_drain( + async fn execute_drain( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], + params: &[DbParam<'_>], ) -> Result<(), TpccError> { let mut iter = self.execute_raw(statement, params)?; while let Some(row) = iter.next() { @@ -172,22 +174,22 @@ impl<'a> BackendTransaction for SqliteTransaction<'a> { Ok(()) } - fn with_query_one( + async fn with_query_one( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, + params: &[DbParam<'_>], + visitor: RowVisitor<'_>, ) -> Result<(), TpccError> { let mut iter = self.execute_raw(statement, params)?; let tuple = iter.next().transpose()?.ok_or(TpccError::EmptyTuples)?; visitor(&tuple.into()) } - fn with_query_all( + async fn with_query_all( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, + params: &[DbParam<'_>], + visitor: RowVisitor<'_>, ) -> Result<(), TpccError> { let iter = self.execute_raw(statement, params)?; for row in iter { @@ -196,12 +198,12 @@ impl<'a> BackendTransaction for SqliteTransaction<'a> { Ok(()) } - fn with_query_nth( + async fn with_query_nth( &mut self, statement: &mut Self::PreparedStatement, - params: &[DbParam], + params: &[DbParam<'_>], n: usize, - visitor: &mut dyn FnMut(&TupleImpl) -> Result<(), TpccError>, + visitor: RowVisitor<'_>, ) -> Result<(), TpccError> { let mut iter = self.execute_raw(statement, params)?; for _ in 0..n { @@ -213,7 +215,7 @@ impl<'a> BackendTransaction for SqliteTransaction<'a> { visitor(&tuple.into()) } - fn commit(mut self) -> Result<(), TpccError> { + async fn commit(mut self) -> Result<(), TpccError> { self.connection.execute("COMMIT")?; self.finished = true; Ok(()) diff --git a/tpcc/src/delivery.rs b/tpcc/src/delivery.rs index f73590ba..5189b2d4 100644 --- a/tpcc/src/delivery.rs +++ b/tpcc/src/delivery.rs @@ -38,7 +38,7 @@ pub(crate) struct DeliveryTest; impl TpccTransaction for Delivery { type Args = DeliveryArgs; - fn run( + async fn run( tx: &mut T, args: &Self::Args, statements: &mut [T::PreparedStatement], @@ -58,7 +58,8 @@ impl TpccTransaction for Delivery { no_o_id = tuple[0].i32().unwrap(); Ok(()) }, - )?; + ) + .await?; if no_o_id == 0 { continue; @@ -71,7 +72,8 @@ impl TpccTransaction for Delivery { (2, DataValue::Int8(d_id as i8).into()), (3, DataValue::Int16(args.w_id as i16).into()), ], - )?; + ) + .await?; // "SELECT o_c_id FROM orders WHERE o_id = ? AND o_d_id = ? AND o_w_id = ?" let mut c_id = 0; tx.with_query_one( @@ -85,7 +87,8 @@ impl TpccTransaction for Delivery { c_id = tuple[0].i32().unwrap(); Ok(()) }, - )?; + ) + .await?; // "UPDATE orders SET o_carrier_id = ? WHERE o_id = ? AND o_d_id = ? AND o_w_id = ?" tx.execute_drain( &mut statements[3], @@ -95,7 +98,8 @@ impl TpccTransaction for Delivery { (3, DataValue::Int8(d_id as i8).into()), (4, DataValue::Int16(args.w_id as i16).into()), ], - )?; + ) + .await?; // "UPDATE order_line SET ol_delivery_d = ? WHERE ol_o_id = ? AND ol_d_id = ? AND ol_w_id = ?" tx.execute_drain( &mut statements[4], @@ -105,7 +109,8 @@ impl TpccTransaction for Delivery { (3, DataValue::Int8(d_id as i8).into()), (4, DataValue::Int16(args.w_id as i16).into()), ], - )?; + ) + .await?; // "SELECT SUM(ol_amount) FROM order_line WHERE ol_o_id = ? AND ol_d_id = ? AND ol_w_id = ?" let mut ol_total = Default::default(); tx.with_query_one( @@ -119,7 +124,8 @@ impl TpccTransaction for Delivery { ol_total = tuple[0].decimal().unwrap(); Ok(()) }, - )?; + ) + .await?; // "UPDATE customer SET c_balance = c_balance + ? , c_delivery_cnt = c_delivery_cnt + 1 WHERE c_id = ? AND c_d_id = ? AND c_w_id = ?" tx.execute_drain( &mut statements[6], @@ -129,7 +135,8 @@ impl TpccTransaction for Delivery { (3, DataValue::Int8(d_id as i8).into()), (4, DataValue::Int16(args.w_id as i16).into()), ], - )?; + ) + .await?; } Ok(()) @@ -137,19 +144,19 @@ impl TpccTransaction for Delivery { } impl TpccTest for DeliveryTest { - fn do_transaction( + async fn do_transaction( &self, rng: &mut ThreadRng, tx: &mut T, - num_ware: usize, - _: &TpccArgs, + _: usize, + args: &TpccArgs, statements: &mut [T::PreparedStatement], ) -> Result<(), TpccError> { - let w_id = rng.gen_range(0..num_ware) + 1; + let w_id = args.home_ware; let o_carrier_id = rng.gen_range(1..10); let args = DeliveryArgs::new(w_id, o_carrier_id); - Delivery::run(tx, &args, statements)?; + Delivery::run(tx, &args, statements).await?; Ok(()) } diff --git a/tpcc/src/load.rs b/tpcc/src/load.rs index f889d678..60922f3a 100644 --- a/tpcc/src/load.rs +++ b/tpcc/src/load.rs @@ -65,7 +65,7 @@ impl<'a, E: SimpleExecutor> SqlBatch<'a, E> { } } - fn push(&mut self, statement: &str) -> Result<(), TpccError> { + async fn push(&mut self, statement: &str) -> Result<(), TpccError> { if self.pending != 0 { self.sql.push(';'); } @@ -73,20 +73,20 @@ impl<'a, E: SimpleExecutor> SqlBatch<'a, E> { self.pending += 1; if self.pending >= LOAD_BATCH_SIZE { - self.flush()?; + self.flush().await?; } Ok(()) } - fn flush(&mut self) -> Result<(), TpccError> { + async fn flush(&mut self) -> Result<(), TpccError> { if self.pending == 0 { return Ok(()); } let sql = std::mem::take(&mut self.sql); self.pending = 0; - self.exec.execute_batch(&sql) + self.exec.execute_batch(&sql).await } } @@ -149,11 +149,11 @@ impl Load { /// i_data varchar(50) /// /// primary key (i_id) - pub fn load_items( + pub async fn load_items( rng: &mut ThreadRng, exec: &mut impl SimpleExecutor, ) -> Result<(), TpccError> { - exec.execute_batch("drop table if exists item;")?; + exec.execute_batch("drop table if exists item;").await?; exec.execute_batch( "create table item ( i_id int not null, @@ -162,7 +162,8 @@ impl Load { i_price decimal(5,2), i_data varchar(50), PRIMARY KEY(i_id) );", - )?; + ) + .await?; let pb = ProgressBar::new(MAX_ITEMS as u64); pb.set_style( ProgressStyle::default_bar() @@ -189,15 +190,17 @@ impl Load { i_data = format!("{}original{}", prefix, remainder); } - batch.push(&format!( + batch + .push(&format!( "insert into item values ({i_id}, {i_im_id}, '{i_name}', {i_price}, '{i_data}')" - ))?; + )) + .await?; pb.set_position(i_id as u64); } - batch.flush()?; + batch.flush().await?; finish_progress(&pb, Some("Items loaded")); println!("[Analyze Table: item]"); - exec.execute_batch("analyze table item")?; + exec.execute_batch("analyze table item").await?; Ok(()) } @@ -214,12 +217,13 @@ impl Load { /// w_ytd decimal(12, 2) /// /// primary key (w_id) - pub fn load_warehouses( + pub async fn load_warehouses( rng: &mut ThreadRng, exec: &mut impl SimpleExecutor, num_ware: usize, ) -> Result<(), TpccError> { - exec.execute_batch("drop table if exists warehouse;")?; + exec.execute_batch("drop table if exists warehouse;") + .await?; exec.execute_batch( "create table warehouse ( w_id smallint not null, @@ -232,8 +236,9 @@ impl Load { w_tax decimal(4,2), w_ytd decimal(12,2), PRIMARY KEY(w_id) );", - )?; - exec.execute_batch("drop table if exists stock;")?; + ) + .await?; + exec.execute_batch("drop table if exists stock;").await?; exec.execute_batch( "create table stock ( s_i_id int not null, @@ -254,9 +259,11 @@ impl Load { s_remote_cnt smallint, s_data varchar(50), PRIMARY KEY(s_w_id, s_i_id) );", - )?; - exec.execute_batch("CREATE INDEX fkey_stock_2 ON stock (s_i_id);")?; - exec.execute_batch("drop table if exists district;")?; + ) + .await?; + exec.execute_batch("CREATE INDEX fkey_stock_2 ON stock (s_i_id);") + .await?; + exec.execute_batch("drop table if exists district;").await?; exec.execute_batch( "create table district ( d_id tinyint not null, @@ -271,7 +278,8 @@ impl Load { d_ytd decimal(12,2), d_next_o_id int, primary key (d_w_id, d_id) );", - )?; + ) + .await?; let pb = ProgressBar::new(num_ware as u64); pb.set_style( ProgressStyle::default_bar() @@ -297,17 +305,18 @@ impl Load { exec.execute_batch(&format!( "insert into warehouse values({}, '{}', '{}', '{}', '{}', '{}', '{}', {}, {})", w_id, w_name, w_street_1, w_street_2, w_city, w_state, w_zip, w_tax, w_ytd, - ))?; - Self::stock(rng, exec, w_id)?; - Self::district(rng, exec, w_id)?; + )) + .await?; + Self::stock(rng, exec, w_id).await?; + Self::district(rng, exec, w_id).await?; pb.set_position(w_id as u64); } finish_progress(&pb, Some("Warehouses loaded")); println!("[Analyze Table: stock]"); - exec.execute_batch("analyze table stock")?; + exec.execute_batch("analyze table stock").await?; println!("[Analyze Table: district]"); - match exec.execute_batch("analyze table district") { + match exec.execute_batch("analyze table district").await { Ok(_) | Err(TpccError::Database(DatabaseError::TooManyBuckets(..))) => (), err => return err, }; @@ -315,12 +324,12 @@ impl Load { Ok(()) } - pub fn load_custs( + pub async fn load_custs( rng: &mut ThreadRng, exec: &mut impl SimpleExecutor, num_ware: usize, ) -> Result<(), TpccError> { - exec.execute_batch("drop table if exists customer;")?; + exec.execute_batch("drop table if exists customer;").await?; exec.execute_batch( "create table customer ( c_id int not null, @@ -345,11 +354,11 @@ impl Load { c_delivery_cnt smallint, c_data text, PRIMARY KEY(c_w_id, c_d_id, c_id) );", - )?; - exec.execute_batch( - "CREATE INDEX idx_customer ON customer (c_w_id,c_d_id,c_last,c_first);", - )?; - exec.execute_batch("drop table if exists history;")?; + ) + .await?; + exec.execute_batch("CREATE INDEX idx_customer ON customer (c_w_id,c_d_id,c_last,c_first);") + .await?; + exec.execute_batch("drop table if exists history;").await?; exec.execute_batch( "create table history ( h_c_id int, @@ -361,7 +370,8 @@ impl Load { h_amount decimal(6,2), h_data varchar(24), PRIMARY KEY(h_c_id, h_c_d_id, h_c_w_id, h_d_id, h_w_id, h_date) );", - )?; + ) + .await?; for w_id in 1..num_ware + 1 { for d_id in 1..DIST_PER_WARE + 1 { log_phase( @@ -370,21 +380,21 @@ impl Load { DIST_PER_WARE, &format!("Warehouse {}/{}", w_id, num_ware), ); - Self::load_customers(rng, exec, d_id, w_id)?; + Self::load_customers(rng, exec, d_id, w_id).await?; } } println!("[Analyze Table: customer]"); - exec.execute_batch("analyze table customer")?; + exec.execute_batch("analyze table customer").await?; Ok(()) } - pub fn load_ord( + pub async fn load_ord( rng: &mut ThreadRng, exec: &mut impl SimpleExecutor, num_ware: usize, ) -> Result<(), TpccError> { - exec.execute_batch("drop table if exists orders;")?; + exec.execute_batch("drop table if exists orders;").await?; exec.execute_batch( "create table orders ( o_id int not null, @@ -396,17 +406,22 @@ impl Load { o_ol_cnt tinyint, o_all_local tinyint, PRIMARY KEY(o_w_id, o_d_id, o_id) );", - )?; - exec.execute_batch("CREATE INDEX idx_orders ON orders (o_w_id,o_d_id,o_c_id,o_id);")?; - exec.execute_batch("drop table if exists new_orders;")?; + ) + .await?; + exec.execute_batch("CREATE INDEX idx_orders ON orders (o_w_id,o_d_id,o_c_id,o_id);") + .await?; + exec.execute_batch("drop table if exists new_orders;") + .await?; exec.execute_batch( "create table new_orders ( no_o_id int not null, no_d_id tinyint not null, no_w_id smallint not null, PRIMARY KEY(no_w_id, no_d_id, no_o_id));", - )?; - exec.execute_batch("drop table if exists order_line;")?; + ) + .await?; + exec.execute_batch("drop table if exists order_line;") + .await?; exec.execute_batch( "create table order_line ( ol_o_id int not null, @@ -420,13 +435,16 @@ impl Load { ol_amount decimal(6,2), ol_dist_info char(24), PRIMARY KEY(ol_w_id, ol_d_id, ol_o_id, ol_number) );", - )?; + ) + .await?; exec.execute_batch( "CREATE INDEX fkey_order_line_1 ON order_line (ol_o_id, ol_d_id, ol_w_id);", - )?; + ) + .await?; exec.execute_batch( "CREATE INDEX fkey_order_line_2 ON order_line (ol_supply_w_id, ol_i_id);", - )?; + ) + .await?; for w_id in 1..num_ware + 1 { for d_id in 1..DIST_PER_WARE + 1 { log_phase( @@ -435,13 +453,13 @@ impl Load { DIST_PER_WARE, &format!("Warehouse {}/{}", w_id, num_ware), ); - Self::load_orders(rng, exec, d_id, w_id)?; + Self::load_orders(rng, exec, d_id, w_id).await?; } } println!("[Analyze Table: orders & order_line & new_order]"); - exec.execute_batch("analyze table orders")?; - exec.execute_batch("analyze table order_line")?; - exec.execute_batch("analyze table new_orders")?; + exec.execute_batch("analyze table orders").await?; + exec.execute_batch("analyze table order_line").await?; + exec.execute_batch("analyze table new_orders").await?; Ok(()) } @@ -467,7 +485,7 @@ impl Load { /// s_data varchar(50) /// /// primary key(s_w_id, s_i_id) - pub fn stock( + pub async fn stock( rng: &mut ThreadRng, exec: &mut impl SimpleExecutor, w_id: usize, @@ -521,10 +539,10 @@ impl Load { 0, 0, s_data, - ))?; + )).await?; pb.set_position(s_i_id as u64); } - batch.flush()?; + batch.flush().await?; finish_progress(&pb, None); Ok(()) @@ -564,7 +582,7 @@ impl Load { /// /// /// primary key (d_w_id, d_id) - pub fn district( + pub async fn district( rng: &mut ThreadRng, exec: &mut impl SimpleExecutor, w_id: usize, @@ -607,10 +625,10 @@ impl Load { d_tax, d_ytd, d_next_o_id, - ))?; + )).await?; pb.set_position(d_id as u64); } - batch.flush()?; + batch.flush().await?; finish_progress(&pb, None); Ok(()) @@ -653,7 +671,7 @@ impl Load { /// h_date timestamp(6) /// h_amount decimal(6, 2) /// h_data varchar(24) - pub fn load_customers( + pub async fn load_customers( rng: &mut ThreadRng, exec: &mut impl SimpleExecutor, d_id: usize, @@ -729,19 +747,21 @@ impl Load { c_payment_cnt, c_delivery_cnt, c_data, - ))?; + )).await?; let h_date = &history_date; let h_amount = Decimal::from_f64_retain(10.0).unwrap().round_dp(2); let h_data = generate_string(rng, 12, 24); - batch.push(&format!( - "insert into history values({}, {}, {}, {}, {}, '{}', {}, '{}')", - c_id, c_d_id, c_w_id, c_d_id, c_w_id, h_date, h_amount, h_data, - ))?; + batch + .push(&format!( + "insert into history values({}, {}, {}, {}, {}, '{}', {}, '{}')", + c_id, c_d_id, c_w_id, c_d_id, c_w_id, h_date, h_amount, h_data, + )) + .await?; pb.set_position(c_id as u64); } - batch.flush()?; + batch.flush().await?; finish_progress(&pb, None); Ok(()) @@ -784,7 +804,7 @@ impl Load { /// ol_dist_info char(24) /// /// primary key(ol_w_id, ol_d_id, ol_o_id, ol_number) - pub fn load_orders( + pub async fn load_orders( rng: &mut ThreadRng, exec: &mut impl SimpleExecutor, d_id: usize, @@ -812,18 +832,22 @@ impl Load { let date = format!("'{}'", Utc::now().format("%Y-%m-%d %H:%M:%S")); let o_carrier_id = if o_id > 2100 { - batch.push(&format!( - "insert into new_orders values({}, {}, {})", - o_id, o_d_id, o_w_id, - ))?; + batch + .push(&format!( + "insert into new_orders values({}, {}, {})", + o_id, o_d_id, o_w_id, + )) + .await?; "null".to_string() } else { o_carrier_id.to_string() }; - batch.push(&format!( - "insert into orders values({}, {}, {}, {}, {}, {}, {}, {})", - o_id, o_d_id, o_w_id, o_c_id, date, o_carrier_id, o_ol_cnt, "1", - ))?; + batch + .push(&format!( + "insert into orders values({}, {}, {}, {}, {}, {}, {}, {})", + o_id, o_d_id, o_w_id, o_c_id, date, o_carrier_id, o_ol_cnt, "1", + )) + .await?; for ol in 1..o_ol_cnt + 1 { let ol_i_id = rng.gen_range(1..MAX_ITEMS); @@ -836,23 +860,25 @@ impl Load { } else { (date.as_str(), format!("{:.2}", rng.gen_range(0.1..100.0))) }; - batch.push(&format!( - "insert into order_line values({}, {}, {}, {}, {}, {}, {}, {}, {}, '{}')", - o_id, - o_d_id, - o_w_id, - ol, - ol_i_id, - ol_supply_w_id, - ol_delivery_d, - ol_quantity, - ol_amount, - ol_dist_info, - ))?; + batch + .push(&format!( + "insert into order_line values({}, {}, {}, {}, {}, {}, {}, {}, {}, '{}')", + o_id, + o_d_id, + o_w_id, + ol, + ol_i_id, + ol_supply_w_id, + ol_delivery_d, + ol_quantity, + ol_amount, + ol_dist_info, + )) + .await?; } pb.set_position((o_id - 1) as u64); } - batch.flush()?; + batch.flush().await?; finish_progress(&pb, None); Ok(()) @@ -873,20 +899,20 @@ mod tests { } impl SimpleExecutor for RecordingExecutor { - fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError> { + async fn execute_batch(&mut self, sql: &str) -> Result<(), TpccError> { self.calls.borrow_mut().push(sql.to_string()); Ok(()) } } - #[test] - fn sql_batch_groups_statements() { + #[tokio::test] + async fn sql_batch_groups_statements() { let mut exec = RecordingExecutor::default(); let mut batch = SqlBatch::new(&mut exec); - batch.push("insert 1").unwrap(); - batch.push("insert 2").unwrap(); - batch.flush().unwrap(); + batch.push("insert 1").await.unwrap(); + batch.push("insert 2").await.unwrap(); + batch.flush().await.unwrap(); assert_eq!( exec.calls.into_inner(), @@ -894,15 +920,15 @@ mod tests { ); } - #[test] - fn sql_batch_flushes_at_batch_size() { + #[tokio::test] + async fn sql_batch_flushes_at_batch_size() { let mut exec = RecordingExecutor::default(); let mut batch = SqlBatch::new(&mut exec); for i in 0..=LOAD_BATCH_SIZE { - batch.push(&format!("insert {i}")).unwrap(); + batch.push(&format!("insert {i}")).await.unwrap(); } - batch.flush().unwrap(); + batch.flush().await.unwrap(); let calls = exec.calls.into_inner(); assert_eq!(calls.len(), 2); diff --git a/tpcc/src/main.rs b/tpcc/src/main.rs index 588751ea..34dff38d 100644 --- a/tpcc/src/main.rs +++ b/tpcc/src/main.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(feature = "kitesql-sync")] use crate::backend::dual::DualBackend; -use crate::backend::kitesql_lmdb::KiteSqlLmdbBackend; -use crate::backend::kitesql_rocksdb::{KiteSqlOptimisticRocksDbBackend, KiteSqlRocksDbBackend}; +use crate::backend::kitesql::KiteSqlBackend; use crate::backend::sqlite::SqliteBackend; use crate::backend::{BackendControl, BackendTransaction, StatementSpec}; use crate::delivery::DeliveryTest; @@ -28,6 +28,8 @@ use crate::utils::SeqGen; use clap::{Parser, ValueEnum}; use indicatif::{ProgressBar, ProgressStyle}; use kite_sql::errors::DatabaseError; +#[cfg(feature = "slatedb-s3")] +use kite_sql::storage::slatedb::{SlateDbCacheOptions, SlateDbDiskCache}; use kite_sql::types::LogicalType; #[cfg(all(unix, feature = "pprof"))] use pprof::ProfilerGuard; @@ -38,10 +40,11 @@ use std::error::Error; use std::fmt; use std::fs; use std::path::Path; -#[cfg(any(feature = "pprof", test))] +#[cfg(any(feature = "pprof", feature = "slatedb-s3", test))] use std::path::PathBuf; use std::thread; use std::time::{Duration, Instant}; +use tokio::runtime::Runtime; mod backend; mod delivery; @@ -74,7 +77,7 @@ const TX_NAMES: [&str; 5] = [ pub(crate) trait TpccTransaction { type Args; - fn run( + async fn run( tx: &mut T, args: &Self::Args, statements: &mut [T::PreparedStatement], @@ -82,7 +85,7 @@ pub(crate) trait TpccTransaction { } pub(crate) trait TpccTest { - fn do_transaction( + async fn do_transaction( &self, rng: &mut ThreadRng, tx: &mut T, @@ -94,6 +97,7 @@ pub(crate) trait TpccTest { struct TpccArgs { joins: bool, + home_ware: usize, } #[derive(Parser, Debug)] @@ -103,7 +107,14 @@ struct Args { joins: bool, #[clap(long, default_value = "kite_sql_tpcc")] path: String, - #[clap(long, value_enum, default_value = "kitesql-lmdb")] + #[cfg_attr( + feature = "kitesql-sync", + clap(long, value_enum, default_value = "kitesql-lmdb") + )] + #[cfg_attr( + not(feature = "kitesql-sync"), + clap(long, value_enum, default_value = "kitesql-slatedb-s3") + )] backend: BackendKind, #[clap(long, default_value = "5")] max_retry: usize, @@ -111,13 +122,36 @@ struct Args { measure_time: u64, #[clap(long, default_value = "1")] num_ware: usize, - /// Concurrent terminals, each on its own connection. #[clap(long, default_value = "1")] threads: usize, + /// Concurrent terminals, `--threads` by default. Thread `t` runs terminals `t`, + /// `t + threads`, ...; terminal `i` has home warehouse `i % num_ware + 1`. More terminals than + /// threads needs an async storage, whose terminals overlap their waits on one thread. + #[clap(long)] + terminals: Option, #[clap(long, default_value = "false")] rocksdb_stats: bool, #[clap(long, default_value = "false")] sync: bool, + #[cfg(feature = "slatedb-s3")] + #[clap(long)] + s3_bucket: Option, + #[cfg(feature = "slatedb-s3")] + #[clap(long)] + s3_endpoint: Option, + #[cfg(feature = "slatedb-s3")] + #[clap(long, default_value = "false")] + s3_allow_http: bool, + #[cfg(feature = "slatedb-s3")] + #[clap(long, default_value = "4096")] + slatedb_memory_cache_mb: u64, + #[cfg(feature = "slatedb-s3")] + #[clap(long, default_value = "kite_sql_tpcc_cache")] + slatedb_disk_cache: PathBuf, + /// 0 turns the disk cache off. Its files grow as they fill, so a large capacity is cheap. + #[cfg(feature = "slatedb-s3")] + #[clap(long, default_value = "65536")] + slatedb_disk_cache_mb: usize, #[cfg(feature = "pprof")] #[clap(long)] pprof_output: Option, @@ -128,48 +162,111 @@ struct Args { #[derive(Copy, Clone, Debug, ValueEnum)] enum BackendKind { + #[cfg(feature = "kitesql-sync")] KitesqlRocksdb, + #[cfg(feature = "kitesql-sync")] KitesqlOptimisticRocksdb, + #[cfg(feature = "kitesql-sync")] KitesqlLmdb, + #[cfg(feature = "slatedb-s3")] + KitesqlSlatedbS3, Sqlite, + #[cfg(feature = "kitesql-sync")] Dual, } fn main() -> Result<(), TpccError> { let args = Args::parse(); let mut rng = rand::thread_rng(); + // The workload is async; each terminal thread drives its transactions with `block_on`. + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; match args.backend { + #[cfg(feature = "kitesql-sync")] BackendKind::KitesqlRocksdb => { reset_db_path(Path::new(&args.path))?; - let mut backend = - KiteSqlRocksDbBackend::new(&args.path, args.rocksdb_stats, args.sync)?; - run_tpcc(&mut backend, &args, &mut rng) + let mut backend = KiteSqlBackend::rocksdb(&args.path, args.rocksdb_stats, args.sync)?; + run_tpcc(&mut backend, &args, &mut rng, &runtime) } + #[cfg(feature = "kitesql-sync")] BackendKind::KitesqlOptimisticRocksdb => { reset_db_path(Path::new(&args.path))?; let mut backend = - KiteSqlOptimisticRocksDbBackend::new(&args.path, args.rocksdb_stats, args.sync)?; - run_tpcc(&mut backend, &args, &mut rng) + KiteSqlBackend::optimistic_rocksdb(&args.path, args.rocksdb_stats, args.sync)?; + run_tpcc(&mut backend, &args, &mut rng, &runtime) } + #[cfg(feature = "kitesql-sync")] BackendKind::KitesqlLmdb => { reset_db_path(Path::new(&args.path))?; - let mut backend = KiteSqlLmdbBackend::new(&args.path, args.sync)?; - run_tpcc(&mut backend, &args, &mut rng) + let mut backend = KiteSqlBackend::lmdb(&args.path, args.sync)?; + run_tpcc(&mut backend, &args, &mut rng, &runtime) + } + #[cfg(feature = "slatedb-s3")] + BackendKind::KitesqlSlatedbS3 => { + let config = + kite_sql::storage::slatedb::S3Config { + endpoint: args.s3_endpoint.clone(), + allow_http: args.s3_allow_http, + ..kite_sql::storage::slatedb::S3Config::new(args.s3_bucket.clone().ok_or_else( + || TpccError::InvalidArgs("--s3-bucket is required".to_string()), + )?) + }; + let run = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|err| TpccError::InvalidArgs(err.to_string()))? + .as_millis(); + let path = format!("{}/{run}", args.path); + println!("[SlateDB] s3://{}/{path}", config.bucket); + let cache = slatedb_cache_options(&args)?; + let storage = runtime + .block_on(kite_sql::storage::slatedb::SlateDbStorage::open_s3( + path.as_str(), + &config, + cache, + ))? + .await_durable(args.sync); + let mut backend = KiteSqlBackend::with_storage(storage.clone())?; + let result = run_tpcc(&mut backend, &args, &mut rng, &runtime); + runtime.block_on(storage.close())?; + result } BackendKind::Sqlite => { reset_db_path(Path::new(&args.path))?; let mut backend = SqliteBackend::new(&args.path, args.sync)?; - run_tpcc(&mut backend, &args, &mut rng) + run_tpcc(&mut backend, &args, &mut rng, &runtime) } + #[cfg(feature = "kitesql-sync")] BackendKind::Dual => { reset_db_path(Path::new(&args.path))?; let mut backend = DualBackend::new(&args.path, args.rocksdb_stats)?; - run_tpcc(&mut backend, &args, &mut rng) + run_tpcc(&mut backend, &args, &mut rng, &runtime) } } } +/// The SlateDB block cache: memory, a local disk tier, and the blocks compactions write, which +/// would otherwise leave the cache as the load compacts. +#[cfg(feature = "slatedb-s3")] +fn slatedb_cache_options(args: &Args) -> Result { + let disk = if args.slatedb_disk_cache_mb == 0 { + None + } else { + // What a previous run left belongs to a database no run reads again. + reset_db_path(&args.slatedb_disk_cache)?; + Some(SlateDbDiskCache { + dir: args.slatedb_disk_cache.clone(), + capacity_bytes: args.slatedb_disk_cache_mb << 20, + }) + }; + Ok(SlateDbCacheOptions { + memory_bytes: Some(args.slatedb_memory_cache_mb << 20), + disk, + cache_compaction_output: true, + }) +} + fn reset_db_path(path: &Path) -> Result<(), TpccError> { if !path.exists() { return Ok(()); @@ -188,34 +285,49 @@ fn run_tpcc( backend: &mut B, args: &Args, rng: &mut ThreadRng, + runtime: &Runtime, ) -> Result<(), TpccError> { - Load::load_items(rng, backend)?; - Load::load_warehouses(rng, backend, args.num_ware)?; - Load::load_custs(rng, backend, args.num_ware)?; - Load::load_ord(rng, backend, args.num_ware)?; + runtime.block_on(async { + Load::load_items(rng, backend).await?; + Load::load_warehouses(rng, backend, args.num_ware).await?; + Load::load_custs(rng, backend, args.num_ware).await?; + Load::load_ord(rng, backend, args.num_ware).await + })?; let backend = &*backend; + let terminals = args.terminals.unwrap_or(args.threads); + if terminals < args.threads { + return Err(TpccError::InvalidArgs(format!( + "--terminals {terminals} is less than --threads {}", + args.threads + ))); + } + if terminals != args.threads && !backend.is_async() { + return Err(TpccError::InvalidArgs( + "more terminals than threads needs an async storage".to_string(), + )); + } let tpcc_start = Instant::now(); #[cfg(all(unix, feature = "pprof"))] let pprof = PprofSession::start(args)?; let counters = thread::scope(|scope| { let threads = (1..args.threads) - .map(|_| { + .map(|thread_id| { let worker = backend.worker()?; Ok(scope.spawn(move || { - measure(worker.borrow(), args, tpcc_start, ProgressBar::hidden()) + runtime.block_on(run_terminals( + worker.borrow(), + args, + thread_id, + terminals, + tpcc_start, + )) })) }) .collect::, TpccError>>()?; - let mut total = measure(backend, args, tpcc_start, ProgressBar::new_spinner())?; + let mut total = runtime.block_on(run_terminals(backend, args, 0, terminals, tpcc_start))?; for thread in threads { - let (success, late, failure, rt_hist) = thread.join().expect("tpcc thread panicked")?; - for i in 0..5 { - total.0[i] += success[i]; - total.1[i] += late[i]; - total.2[i] += failure[i]; - } - total.3.merge(&rt_hist); + total.merge(&thread.join().expect("tpcc thread panicked")?); } Ok::<_, TpccError>(total) })?; @@ -227,11 +339,58 @@ fn run_tpcc( Ok(()) } -type Counters = ([usize; 5], [usize; 5], [usize; 5], RtHist); +struct Counters { + success: [usize; 5], + late: [usize; 5], + failure: [usize; 5], + conflict: [usize; 5], + rt_hist: RtHist, +} + +impl Counters { + fn merge(&mut self, other: &Counters) { + for i in 0..5 { + self.success[i] += other.success[i]; + self.late[i] += other.late[i]; + self.failure[i] += other.failure[i]; + self.conflict[i] += other.conflict[i]; + } + self.rt_hist.merge(&other.rt_hist); + } +} -fn measure( +async fn run_terminals( backend: &B, args: &Args, + thread_id: usize, + terminals: usize, + tpcc_start: Instant, +) -> Result { + let runs = (thread_id..terminals) + .step_by(args.threads) + .map(|terminal_id| { + let progress = if terminal_id == 0 { + ProgressBar::new_spinner() + } else { + ProgressBar::hidden() + }; + measure(backend, args, terminal_id, tpcc_start, progress) + }); + let mut counters = futures::future::try_join_all(runs).await?.into_iter(); + let mut total = counters + .next() + .expect("a thread runs at least one terminal"); + for other in counters { + total.merge(&other); + } + Ok(total) +} + +/// Runs terminal `terminal_id`, whose home warehouse is `terminal_id % num_ware + 1`. +async fn measure( + backend: &B, + args: &Args, + terminal_id: usize, tpcc_start: Instant, progress: ProgressBar, ) -> Result { @@ -243,7 +402,11 @@ fn measure( let mut success = [0usize; 5]; let mut late = [0usize; 5]; let mut failure = [0usize; 5]; - let tpcc_args = TpccArgs { joins: args.joins }; + let mut conflict = [0usize; 5]; + let tpcc_args = TpccArgs { + joins: args.joins, + home_ware: terminal_id % args.num_ware + 1, + }; let duration = Duration::new(args.measure_time, 0); let mut round_count = 0; @@ -263,41 +426,47 @@ fn measure( let i = seq_gen.get(); let statement = &mut test_statements[i]; - let mut is_succeed = false; - let mut last_error = None; - for _attempt in 0..=args.max_retry { - let transaction_start = Instant::now(); + // The response time covers the retries too. + let transaction_start = Instant::now(); + for attempt in 0..=args.max_retry { let mut tx = backend.new_transaction(TX_READ_ONLY[i])?; - if let Err(err) = + let result = async { do_tpcc_transaction::(i, rng, &mut tx, args.num_ware, &tpcc_args, statement) - .and_then(|()| tx.commit()) - { - failure[i] += 1; - last_error = Some(err); - } else { - let rt = transaction_start.elapsed(); - rt_hist.hist_inc(i, rt); - is_succeed = true; - - if rt <= RT_LIMITS[i] { - success[i] += 1; - } else { - late[i] += 1; - } - break; + .await?; + tx.commit().await } - } - if !is_succeed { - if let Some(err) = last_error { - eprintln!( - "[{}] Error after {} retries: {}", - tpcc_test_name(i), - args.max_retry, - err - ); + .await; + match result { + Ok(()) => { + let rt = transaction_start.elapsed(); + rt_hist.hist_inc(i, rt); + + if rt <= RT_LIMITS[i] { + success[i] += 1; + } else { + late[i] += 1; + } + break; + } + Err(err) if attempt == args.max_retry => { + eprintln!( + "[{}] Error after {} retries: {}", + tpcc_test_name(i), + args.max_retry, + err + ); + return Err(TpccError::MaxRetry); + } + Err(err) if err.is_conflict() => { + conflict[i] += 1; + // Backs off for a random, growing while, so that conflicting terminals + // spread out instead of colliding again. + let max_us = 1000u64 << attempt.min(6); + tokio::time::sleep(Duration::from_micros(rng.gen_range(0..=max_us))).await; + } + Err(_) => failure[i] += 1, } - return Err(TpccError::MaxRetry); } update_progress_bar( &progress, @@ -325,16 +494,28 @@ fn measure( round_count += 1; } progress.finish_and_clear(); - Ok((success, late, failure, rt_hist)) + Ok(Counters { + success, + late, + failure, + conflict, + rt_hist, + }) } fn report( backend: &B, - (success, late, failure, rt_hist): Counters, + Counters { + success, + late, + failure, + conflict, + rt_hist, + }: Counters, tpcc_start: Instant, ) -> Result<(), TpccError> { let actual_tpcc_time = tpcc_start.elapsed(); - print_summary_table(&success, &late, &failure, actual_tpcc_time); + print_summary_table(&success, &late, &failure, &conflict, actual_tpcc_time); print_constraint_checks(&success, &late); print_response_checks(&success, &late); println!(); @@ -354,7 +535,7 @@ fn tpcc_test_name(index: usize) -> &'static str { TX_NAMES[index] } -fn do_tpcc_transaction<'a, B: BackendControl>( +async fn do_tpcc_transaction<'a, B: BackendControl>( index: usize, rng: &mut ThreadRng, tx: &mut B::Transaction<'a>, @@ -363,11 +544,31 @@ fn do_tpcc_transaction<'a, B: BackendControl>( statements: &mut [B::PreparedStatement<'a>], ) -> Result<(), TpccError> { match index { - 0 => NewOrdTest.do_transaction(rng, tx, num_ware, args, statements), - 1 => PaymentTest.do_transaction(rng, tx, num_ware, args, statements), - 2 => OrderStatTest.do_transaction(rng, tx, num_ware, args, statements), - 3 => DeliveryTest.do_transaction(rng, tx, num_ware, args, statements), - 4 => SlevTest.do_transaction(rng, tx, num_ware, args, statements), + 0 => { + NewOrdTest + .do_transaction(rng, tx, num_ware, args, statements) + .await + } + 1 => { + PaymentTest + .do_transaction(rng, tx, num_ware, args, statements) + .await + } + 2 => { + OrderStatTest + .do_transaction(rng, tx, num_ware, args, statements) + .await + } + 3 => { + DeliveryTest + .do_transaction(rng, tx, num_ware, args, statements) + .await + } + 4 => { + SlevTest + .do_transaction(rng, tx, num_ware, args, statements) + .await + } _ => Err(TpccError::InvalidTransaction), } } @@ -693,23 +894,29 @@ fn stmt( } } -fn print_summary_table(success: &[usize], late: &[usize], failure: &[usize], elapsed: Duration) { - println!("---------------------------------------------------"); +fn print_summary_table( + success: &[usize], + late: &[usize], + failure: &[usize], + conflict: &[usize], + elapsed: Duration, +) { + println!("--------------------------------------------------------------"); println!( "Transaction Summary (elapsed {:.1}s)", elapsed.as_secs_f32() ); - println!("+--------------+---------+------+---------+-------+"); - println!("| Transaction | Success | Late | Failure | Total |"); - println!("+--------------+---------+------+---------+-------+"); + println!("+--------------+---------+------+---------+----------+-------+"); + println!("| Transaction | Success | Late | Failure | Conflict | Total |"); + println!("+--------------+---------+------+---------+----------+-------+"); for (idx, name) in TX_NAMES.iter().enumerate() { - let total = success[idx] + late[idx] + failure[idx]; + let total = success[idx] + late[idx] + failure[idx] + conflict[idx]; println!( - "| {:<12} | {:>7} | {:>4} | {:>7} | {:>5} |", - name, success[idx], late[idx], failure[idx], total + "| {:<12} | {:>7} | {:>4} | {:>7} | {:>8} | {:>5} |", + name, success[idx], late[idx], failure[idx], conflict[idx], total ); } - println!("+--------------+---------+------+---------+-------+"); + println!("+--------------+---------+------+---------+----------+-------+"); } fn print_constraint_checks(success: &[usize], late: &[usize]) { @@ -844,6 +1051,7 @@ pub enum TpccError { InvalidTransaction, InvalidParameter, InvalidDateTime, + InvalidArgs(String), BackendMismatch(String), Profile(String), } @@ -862,12 +1070,19 @@ impl fmt::Display for TpccError { Self::InvalidTransaction => f.write_str("invalid transaction index"), Self::InvalidParameter => f.write_str("invalid parameter name"), Self::InvalidDateTime => f.write_str("invalid datetime value"), + Self::InvalidArgs(value) => write!(f, "invalid arguments: {value}"), Self::BackendMismatch(value) => write!(f, "backend mismatch: {value}"), Self::Profile(value) => write!(f, "profile error: {value}"), } } } +impl TpccError { + fn is_conflict(&self) -> bool { + matches!(self, Self::Database(err) if err.is_conflict()) + } +} + impl Error for TpccError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { diff --git a/tpcc/src/new_ord.rs b/tpcc/src/new_ord.rs index 4ff00f25..3afe44fe 100644 --- a/tpcc/src/new_ord.rs +++ b/tpcc/src/new_ord.rs @@ -67,7 +67,7 @@ pub(crate) struct NewOrdTest; impl TpccTransaction for NewOrd { type Args = NewOrdArgs; - fn run( + async fn run( tx: &mut T, args: &Self::Args, statements: &mut [T::PreparedStatement], @@ -100,7 +100,8 @@ impl TpccTransaction for NewOrd { w_tax = tuple[3].decimal().unwrap(); Ok(()) }, - )?; + ) + .await?; (c_discount, c_last, c_credit, w_tax) } else { @@ -121,7 +122,8 @@ impl TpccTransaction for NewOrd { c_credit = tuple[2].utf8().unwrap().to_string(); Ok(()) }, - )?; + ) + .await?; // "SELECT w_tax FROM warehouse WHERE w_id = ?" let mut w_tax = Decimal::default(); tx.with_query_one( @@ -131,7 +133,8 @@ impl TpccTransaction for NewOrd { w_tax = tuple[0].decimal().unwrap(); Ok(()) }, - )?; + ) + .await?; (c_discount, c_last, c_credit, w_tax) }; @@ -149,7 +152,8 @@ impl TpccTransaction for NewOrd { d_tax = tuple[1].decimal().unwrap(); Ok(()) }, - )?; + ) + .await?; // "UPDATE district SET d_next_o_id = ? + 1 WHERE d_id = ? AND d_w_id = ?" tx.execute_drain( &mut statements[4], @@ -158,7 +162,8 @@ impl TpccTransaction for NewOrd { (2, DataValue::Int8(args.d_id as i8).into()), (3, DataValue::Int16(args.w_id as i16).into()), ], - )?; + ) + .await?; let o_id = d_next_o_id; // "INSERT INTO orders (o_id, o_d_id, o_w_id, o_c_id, o_entry_d, o_ol_cnt, o_all_local) VALUES(?, ?, ?, ?, ?, ?, ?)" tx.execute_drain( @@ -172,7 +177,8 @@ impl TpccTransaction for NewOrd { (6, DataValue::Int8(args.o_ol_cnt as i8).into()), (7, DataValue::Int8(args.o_all_local as i8).into()), ], - )?; + ) + .await?; // "INSERT INTO new_orders (no_o_id, no_d_id, no_w_id) VALUES (?,?,?)" tx.execute_drain( &mut statements[6], @@ -181,7 +187,8 @@ impl TpccTransaction for NewOrd { (2, DataValue::Int8(args.d_id as i8).into()), (3, DataValue::Int16(args.w_id as i16).into()), ], - )?; + ) + .await?; let mut ol_num_seq = vec![0; MAX_NUM_ITEMS]; for i in 0..args.o_ol_cnt { @@ -219,7 +226,8 @@ impl TpccTransaction for NewOrd { i_name = tuple[1].utf8().unwrap().to_string(); i_data = tuple[2].utf8().unwrap().to_string(); Ok(()) - })?; + }) + .await?; price[ol_num_seq[ol_number - 1]] = i_price; iname[ol_num_seq[ol_number - 1]] = i_name; @@ -255,7 +263,8 @@ impl TpccTransaction for NewOrd { s_dist_09 = tuple[10].utf8().unwrap().to_string(); s_dist_10 = tuple[11].utf8().unwrap().to_string(); Ok(()) - })?; + }) + .await?; let ol_dist_info = pick_dist_info( args.d_id, &s_dist_01, &s_dist_02, &s_dist_03, &s_dist_04, &s_dist_05, &s_dist_06, @@ -280,7 +289,7 @@ impl TpccTransaction for NewOrd { (2, DataValue::Int32(ol_i_id as i32).into()), (3, DataValue::Int16(ol_supply_w_id as i16).into()), ]; - tx.execute_drain(&mut statements[9], ¶ms)?; + tx.execute_drain(&mut statements[9], ¶ms).await?; // Tips: Integers always have 7 digits, so divide by 10 here let mut ol_amount = Decimal::from(ol_quantity) @@ -304,7 +313,7 @@ impl TpccTransaction for NewOrd { (8, DataValue::Decimal(ol_amount.round_dp(2)).into()), (9, DataValue::from(ol_dist_info).into()), ]; - tx.execute_drain(&mut statements[10], ¶ms)?; + tx.execute_drain(&mut statements[10], ¶ms).await?; } Ok(()) @@ -312,7 +321,7 @@ impl TpccTransaction for NewOrd { } impl TpccTest for NewOrdTest { - fn do_transaction( + async fn do_transaction( &self, rng: &mut ThreadRng, tx: &mut T, @@ -327,8 +336,8 @@ impl TpccTest for NewOrdTest { let mut supware = vec![0; MAX_NUM_ITEMS]; let mut qty = vec![0; MAX_NUM_ITEMS]; - let w_id = rng.gen_range(0..num_ware) + 1; - let d_id = rng.gen_range(1..DIST_PER_WARE); + let w_id = args.home_ware; + let d_id = rng.gen_range(1..=DIST_PER_WARE); let c_id = nu_rand(rng, 1023, 1, CUST_PER_DIST); let ol_cnt = rng.gen_range(5..15); let rbk = rng.gen_range(1..100); @@ -353,7 +362,7 @@ impl TpccTest for NewOrdTest { let args = NewOrdArgs::new( args.joins, w_id, d_id, c_id, ol_cnt, all_local, itemid, supware, qty, ); - NewOrd::run(tx, &args, statements)?; + NewOrd::run(tx, &args, statements).await?; Ok(()) } diff --git a/tpcc/src/order_stat.rs b/tpcc/src/order_stat.rs index f4fadd3e..16ea9399 100644 --- a/tpcc/src/order_stat.rs +++ b/tpcc/src/order_stat.rs @@ -52,7 +52,7 @@ pub(crate) struct OrderStatTest; impl TpccTransaction for OrderStat { type Args = OrderStatArgs; - fn run( + async fn run( tx: &mut T, args: &Self::Args, statements: &mut [T::PreparedStatement], @@ -71,7 +71,8 @@ impl TpccTransaction for OrderStat { name_cnt = tuple[0].i32().unwrap() as usize; Ok(()) }, - )?; + ) + .await?; // "SELECT c_balance, c_first, c_middle, c_last FROM customer WHERE c_w_id = ? AND c_d_id = ? AND c_last = ? ORDER BY c_first" let params = [ (1, DataValue::Int16(args.w_id as i16).into()), @@ -92,7 +93,8 @@ impl TpccTransaction for OrderStat { c_middle = tuple[2].utf8().unwrap().to_string(); c_last = tuple[3].utf8().unwrap().to_string(); Ok(()) - })?; + }) + .await?; (c_balance, c_first, c_middle, c_last) } else { // "SELECT c_balance, c_first, c_middle, c_last FROM customer WHERE c_w_id = ? AND c_d_id = ? AND c_id = ?" @@ -114,7 +116,8 @@ impl TpccTransaction for OrderStat { c_last = tuple[3].utf8().unwrap().to_string(); Ok(()) }, - )?; + ) + .await?; (c_balance, c_first, c_middle, c_last) }; // "SELECT o_id, o_entry_d, COALESCE(o_carrier_id,0) FROM orders WHERE o_w_id = ? AND o_d_id = ? AND o_c_id = ? AND o_id = (SELECT MAX(o_id) FROM orders WHERE o_w_id = ? AND o_d_id = ? AND o_c_id = ?)" @@ -130,14 +133,16 @@ impl TpccTransaction for OrderStat { tx.with_query_one(&mut statements[3], ¶ms, &mut |tuple| { o_id = tuple[0].i32().unwrap(); Ok(()) - })?; + }) + .await?; // "SELECT ol_i_id, ol_supply_w_id, ol_quantity, ol_amount, ol_delivery_d FROM order_line WHERE ol_w_id = ? AND ol_d_id = ? AND ol_o_id = ?" let params = [ (1, DataValue::Int16(args.w_id as i16).into()), (2, DataValue::Int8(args.d_id as i8).into()), (3, DataValue::Int32(o_id).into()), ]; - tx.with_query_one(&mut statements[4], ¶ms, &mut |_| Ok(()))?; + tx.with_query_one(&mut statements[4], ¶ms, &mut |_| Ok(())) + .await?; // let ol_i_id = tuple[0].i32(); // let ol_supply_w_id = tuple[1].i16(); // let ol_quantity = tuple[2].i8(); @@ -149,22 +154,22 @@ impl TpccTransaction for OrderStat { } impl TpccTest for OrderStatTest { - fn do_transaction( + async fn do_transaction( &self, rng: &mut ThreadRng, tx: &mut T, - num_ware: usize, - _: &TpccArgs, + _: usize, + args: &TpccArgs, statements: &mut [T::PreparedStatement], ) -> Result<(), TpccError> { - let w_id = rng.gen_range(0..num_ware) + 1; - let d_id = rng.gen_range(1..DIST_PER_WARE); + let w_id = args.home_ware; + let d_id = rng.gen_range(1..=DIST_PER_WARE); let c_id = nu_rand(rng, 1023, 1, CUST_PER_DIST); let c_last = last_name(nu_rand(rng, 255, 0, 999)); let by_name = rng.gen_range(1..100) <= 60; let args = OrderStatArgs::new(w_id, d_id, by_name, c_id, c_last); - OrderStat::run(tx, &args, statements)?; + OrderStat::run(tx, &args, statements).await?; Ok(()) } diff --git a/tpcc/src/payment.rs b/tpcc/src/payment.rs index d07a4cee..d9e027e1 100644 --- a/tpcc/src/payment.rs +++ b/tpcc/src/payment.rs @@ -65,7 +65,7 @@ impl TpccTransaction for Payment { type Args = PaymentArgs; #[allow(unused_variables)] - fn run( + async fn run( tx: &mut T, args: &Self::Args, statements: &mut [T::PreparedStatement], @@ -78,7 +78,8 @@ impl TpccTransaction for Payment { (1, DataValue::Decimal(args.h_amount).into()), (2, DataValue::Int16(args.w_id as i16).into()), ], - )?; + ) + .await?; // "SELECT w_street_1, w_street_2, w_city, w_state, w_zip, w_name FROM warehouse WHERE w_id = ?" let mut w_street_1 = String::new(); let mut w_street_2 = String::new(); @@ -98,7 +99,8 @@ impl TpccTransaction for Payment { w_name = tuple[5].utf8().unwrap().to_string(); Ok(()) }, - )?; + ) + .await?; // "UPDATE district SET d_ytd = d_ytd + ? WHERE d_w_id = ? AND d_id = ?" tx.execute_drain( @@ -108,7 +110,8 @@ impl TpccTransaction for Payment { (2, DataValue::Int16(args.w_id as i16).into()), (3, DataValue::Int8(args.d_id as i8).into()), ], - )?; + ) + .await?; // "SELECT d_street_1, d_street_2, d_city, d_state, d_zip, d_name FROM district WHERE d_w_id = ? AND d_id = ?" let mut d_street_1 = String::new(); @@ -132,7 +135,8 @@ impl TpccTransaction for Payment { d_name = tuple[5].utf8().unwrap().to_string(); Ok(()) }, - )?; + ) + .await?; let mut c_id = args.c_id as i32; if args.by_name { @@ -149,7 +153,8 @@ impl TpccTransaction for Payment { name_cnt = tuple[0].i32().unwrap(); Ok(()) }, - )?; + ) + .await?; // "SELECT c_id FROM customer WHERE c_w_id = ? AND c_d_id = ? AND c_last = ? ORDER BY c_first" let params = [ (1, DataValue::Int16(args.c_w_id as i16).into()), @@ -163,7 +168,8 @@ impl TpccTransaction for Payment { tx.with_query_nth(&mut statements[5], ¶ms, target, &mut |tuple| { c_id = tuple[0].i32().unwrap(); Ok(()) - })?; + }) + .await?; } // "SELECT c_first, c_middle, c_last, c_street_1, c_street_2, c_city, c_state, c_zip, c_phone, c_credit, c_credit_lim, c_discount, c_balance, c_since FROM customer WHERE c_w_id = ? AND c_d_id = ? AND c_id = ? FOR UPDATE" let mut c_first = String::new(); @@ -204,7 +210,8 @@ impl TpccTransaction for Payment { c_since = tuple[13].datetime().unwrap(); Ok(()) }, - )?; + ) + .await?; c_balance += args.h_amount; if let Some(c_credit) = c_credit { @@ -222,7 +229,8 @@ impl TpccTransaction for Payment { c_data = tuple[0].utf8().unwrap().to_string(); Ok(()) }, - )?; + ) + .await?; // https://github.com/AgilData/tpcc/blob/dfbabe1e35cc93b2bf2e107fc699eb29c2097e24/src/main/java/com/codefutures/tpcc/Payment.java#L284 // let c_new_data = format!("| {} {} {} {} {} {} {}", c_id, args.c_d_id, args.c_w_id, args.d_id, args.w_id, args.h_amount, ) @@ -237,7 +245,8 @@ impl TpccTransaction for Payment { (4, DataValue::Int8(args.c_d_id as i8).into()), (5, DataValue::Int32(c_id).into()), ], - )?; + ) + .await?; } else { // "UPDATE customer SET c_balance = ? WHERE c_w_id = ? AND c_d_id = ? AND c_id = ?" tx.execute_drain( @@ -248,7 +257,8 @@ impl TpccTransaction for Payment { (3, DataValue::Int8(args.c_d_id as i8).into()), (4, DataValue::Int32(c_id).into()), ], - )?; + ) + .await?; } } else { // "UPDATE customer SET c_balance = ? WHERE c_w_id = ? AND c_d_id = ? AND c_id = ?" @@ -260,7 +270,8 @@ impl TpccTransaction for Payment { (3, DataValue::Int8(args.c_d_id as i8).into()), (4, DataValue::Int32(c_id).into()), ], - )?; + ) + .await?; } let h_data = format!("\\0{d_name} \\0"); // "INSERT INTO history(h_c_d_id, h_c_w_id, h_c_id, h_d_id, h_w_id, h_date, h_amount, h_data) VALUES(?, ?, ?, ?, ?, ?, ?, ?)" @@ -279,23 +290,24 @@ impl TpccTransaction for Payment { (7, DataValue::Decimal(args.h_amount).into()), (8, DataValue::from(h_data).into()), ], - )?; + ) + .await?; Ok(()) } } impl TpccTest for PaymentTest { - fn do_transaction( + async fn do_transaction( &self, rng: &mut ThreadRng, tx: &mut T, num_ware: usize, - _: &TpccArgs, + args: &TpccArgs, statements: &mut [T::PreparedStatement], ) -> Result<(), TpccError> { - let w_id = rng.gen_range(0..num_ware) + 1; - let d_id = rng.gen_range(1..DIST_PER_WARE); + let w_id = args.home_ware; + let d_id = rng.gen_range(1..=DIST_PER_WARE); let c_id = nu_rand(rng, 1023, 1, CUST_PER_DIST); let c_last = last_name(nu_rand(rng, 255, 0, 999)); let h_amount = rng.gen_range(1..5000); @@ -306,7 +318,7 @@ impl TpccTest for PaymentTest { } else { ( other_ware(rng, w_id, num_ware), - rng.gen_range(1..DIST_PER_WARE), + rng.gen_range(1..=DIST_PER_WARE), ) } } else { @@ -322,7 +334,7 @@ impl TpccTest for PaymentTest { c_last, Decimal::from(h_amount), ); - Payment::run(tx, &args, statements)?; + Payment::run(tx, &args, statements).await?; Ok(()) } diff --git a/tpcc/src/slev.rs b/tpcc/src/slev.rs index 50b6da3a..ba9501da 100644 --- a/tpcc/src/slev.rs +++ b/tpcc/src/slev.rs @@ -38,7 +38,7 @@ pub(crate) struct SlevTest; impl TpccTransaction for Slev { type Args = SlevArgs; - fn run( + async fn run( tx: &mut T, args: &Self::Args, statements: &mut [T::PreparedStatement], @@ -55,7 +55,8 @@ impl TpccTransaction for Slev { d_next_o_id = tuple[0].i32().unwrap(); Ok(()) }, - )?; + ) + .await?; // "SELECT DISTINCT ol_i_id FROM order_line WHERE ol_w_id = ? AND ol_d_id = ? AND ol_o_id < ? AND ol_o_id >= (? - 20)" let mut item_ids = Vec::new(); tx.with_query_all( @@ -70,7 +71,8 @@ impl TpccTransaction for Slev { item_ids.push(tuple[0].i32().unwrap()); Ok(()) }, - )?; + ) + .await?; // "SELECT count(*) FROM stock WHERE s_w_id = ? AND s_i_id = ? AND s_quantity < ?" let mut _low_stock = 0; for item_id in item_ids { @@ -85,27 +87,28 @@ impl TpccTransaction for Slev { _low_stock += tuple[0].i32().unwrap(); Ok(()) }, - )?; + ) + .await?; } Ok(()) } } impl TpccTest for SlevTest { - fn do_transaction( + async fn do_transaction( &self, rng: &mut ThreadRng, tx: &mut T, - num_ware: usize, - _: &TpccArgs, + _: usize, + args: &TpccArgs, statements: &mut [T::PreparedStatement], ) -> Result<(), TpccError> { - let w_id = rng.gen_range(0..num_ware) + 1; + let w_id = args.home_ware; let d_id = rng.gen_range(1..=DIST_PER_WARE); let level = rng.gen_range(10..20); let args = SlevArgs::new(w_id, d_id, level); - Slev::run(tx, &args, statements)?; + Slev::run(tx, &args, statements).await?; Ok(()) }