From 97c60b1dfe2eee8cf4e844f8dca55bbb8ff1dc96 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 6 Oct 2026 14:41:18 +0200 Subject: [PATCH] WIP: Add support for configuring Iceberg REST catalog --- Cargo.lock | 442 ++++----- Cargo.toml | 5 +- extra/crds.yaml | 864 ++++++++++++++++++ rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/catalog/commons.rs | 4 +- rust/operator-binary/src/catalog/config.rs | 9 +- rust/operator-binary/src/catalog/iceberg.rs | 121 ++- .../src/controller/dereference.rs | 12 +- .../src/controller/validate.rs | 2 +- rust/operator-binary/src/crd/affinity.rs | 135 +-- .../src/crd/catalog/commons.rs | 2 +- .../src/crd/catalog/delta_lake.rs | 4 +- .../src/crd/catalog/generic.rs | 4 +- rust/operator-binary/src/crd/catalog/hive.rs | 4 +- .../src/crd/catalog/iceberg.rs | 84 +- rust/operator-binary/src/crd/catalog/mod.rs | 234 ++++- rust/operator-binary/src/crd/mod.rs | 2 +- 17 files changed, 1577 insertions(+), 352 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c7ffa6e58..a495b3bce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,7 +169,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -279,9 +279,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "block-buffer" @@ -316,9 +316,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.4.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "jobserver", @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chacha20" @@ -339,7 +339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -356,9 +356,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -366,9 +366,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -378,21 +378,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "colorchoice" @@ -402,9 +402,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -462,6 +462,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -473,36 +479,36 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-bigint" @@ -538,12 +544,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88490bf1b990d87eaaa7ac8aa887f629a08e7359765b4911faf63c3763347d23" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core 0.24.0", - "darling_macro 0.24.0", + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] @@ -561,15 +567,15 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084e274f91c482280130e1e34e0b8d6e66776a060d7b6de7b84289ca778868c4" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" dependencies = [ "ident_case", "proc-macro2", "quote", "strsim", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -585,13 +591,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f5792fa0d41cd2325ce0ffa64f0a340eaebd4971a3a0c5e1ffd2cc488a355e" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core 0.24.0", + "darling_core 0.24.1", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -622,7 +628,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -707,7 +713,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -771,14 +777,14 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] name = "either" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "elliptic-curve" @@ -802,11 +808,16 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] @@ -844,7 +855,7 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -901,9 +912,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "flagset" @@ -913,12 +924,13 @@ checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1004,7 +1016,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -1098,7 +1110,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "libgit2-sys", "log", @@ -1145,9 +1157,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f877e75f39e9827ec50a572dd592684ac28c029578726c85f1b2aa6ab807449" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -1258,9 +1270,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hyper" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -1280,9 +1292,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http", "hyper", @@ -1310,16 +1322,17 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "ipnet", "libc", @@ -1425,9 +1438,9 @@ checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -1467,9 +1480,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -1486,9 +1499,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "is_terminal_polyfill" @@ -1533,9 +1546,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" dependencies = [ "defmt", "jiff-core", @@ -1550,18 +1563,19 @@ dependencies = [ [[package]] name = "jiff-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" dependencies = [ "defmt", + "log", ] [[package]] name = "jiff-static" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" dependencies = [ "jiff-core", "proc-macro2", @@ -1596,7 +1610,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.21", "walkdir", "windows-link", ] @@ -1645,9 +1659,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -1664,20 +1678,20 @@ dependencies = [ "schemars", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] name = "jsonpath-rust" -version = "1.0.8" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e07021eefc09f897611b067ba7897b5e9266edfc902768418968772e5bb06a7" +checksum = "d23892d4caa103325a558250e9f4e406d81cf64f7c01679ae7a64e0413321e51" dependencies = [ "pest", "pest_derive", "regex", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -1706,9 +1720,9 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.119.0#ed84d456ff0b8324db3d7444659cb207564be8c9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Ffrom_docs#5d7902142fe5c7ab0a820117a80b5f4a4e34d5c9" dependencies = [ - "darling 0.24.0", + "darling 0.24.1", "regex", "snafu 0.9.2", ] @@ -1769,7 +1783,7 @@ dependencies = [ "serde", "serde-saphyr", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tower", @@ -1793,7 +1807,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -1831,7 +1845,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tracing", @@ -1839,24 +1853,24 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" dependencies = [ "spin", ] [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libgit2-sys" -version = "0.18.7+1.9.6" +version = "0.18.8+1.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89" +checksum = "7f7c568b25d7489bc3fb2988ed69ab111d2944d2f5fec3d5c987fe545ea97b50" dependencies = [ "cc", "libc", @@ -1899,9 +1913,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "matchers" @@ -1932,9 +1946,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -1942,15 +1956,21 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", "windows-sys 0.61.2", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -1977,7 +1997,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.7", + "rand 0.8.8", "smallvec", "zeroize", ] @@ -2045,7 +2065,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.20", + "thiserror 2.0.21", "tracing", ] @@ -2088,7 +2108,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tonic", "tonic-types", @@ -2126,7 +2146,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", ] @@ -2208,9 +2228,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +checksum = "45d3aca230fad2e6f6317ca0a72724338c4960cb97168a85cdee66df4a9a21a8" dependencies = [ "memchr", "ucd-trie", @@ -2218,9 +2238,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +checksum = "284b60557f2c4a2e72ad3f2d34d42685a2fa4a6a61d0d2a10c0ae2a5e916c2cf" dependencies = [ "pest", "pest_generator", @@ -2228,9 +2248,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +checksum = "1d9d1f08a115309ee99268cf85e5228e0e56aa9caf8841ec12866b6be07c3109" dependencies = [ "pest", "pest_meta", @@ -2241,9 +2261,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +checksum = "ed93ba1a9ffcca32130a5188701c81c0c49cf00d4b7c5007d5148951d743adcb" dependencies = [ "pest", ] @@ -2309,9 +2329,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ "portable-atomic", ] @@ -2327,9 +2347,9 @@ dependencies = [ [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -2422,9 +2442,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "rand_chacha 0.3.1", "rand_core 0.6.4", @@ -2442,9 +2462,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -2501,7 +2521,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -2521,7 +2541,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -2561,11 +2581,11 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-core", @@ -2637,9 +2657,9 @@ dependencies = [ [[package]] name = "rstest" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +checksum = "948203e6d13b83e51a90d7cf236dd58a0065f23f4b902f00f306e7eb2bd09b9c" dependencies = [ "futures-timer", "futures-util", @@ -2648,9 +2668,9 @@ dependencies = [ [[package]] name = "rstest_macros" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +checksum = "a8d92eaf7b6e51e12471d71ddc72608e7151573de44099aacfbb1128177c0da4" dependencies = [ "cfg-if", "glob", @@ -2711,9 +2731,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation", "core-foundation-sys", @@ -2732,15 +2752,15 @@ dependencies = [ [[package]] name = "rustls-platform-verifier-android" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.14" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -2800,7 +2820,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -2838,7 +2858,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation", "core-foundation-sys", "libc", @@ -2921,7 +2941,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -2932,7 +2952,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -3152,14 +3172,14 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.119.0#ed84d456ff0b8324db3d7444659cb207564be8c9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Ffrom_docs#5d7902142fe5c7ab0a820117a80b5f4a4e34d5c9" dependencies = [ "const-oid", "ecdsa", "k8s-openapi", "kube", "p256", - "rand 0.10.2", + "rand 0.10.3", "rand_core 0.6.4", "rsa", "sha2", @@ -3176,7 +3196,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.119.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.119.0#ed84d456ff0b8324db3d7444659cb207564be8c9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Ffrom_docs#5d7902142fe5c7ab0a820117a80b5f4a4e34d5c9" dependencies = [ "base64 0.23.1", "clap", @@ -3193,7 +3213,7 @@ dependencies = [ "json-patch", "k8s-openapi", "kube", - "rand 0.10.2", + "rand 0.10.3", "regex", "schemars", "semver", @@ -3220,18 +3240,18 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.119.0#ed84d456ff0b8324db3d7444659cb207564be8c9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Ffrom_docs#5d7902142fe5c7ab0a820117a80b5f4a4e34d5c9" dependencies = [ - "darling 0.24.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.119.0#ed84d456ff0b8324db3d7444659cb207564be8c9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Ffrom_docs#5d7902142fe5c7ab0a820117a80b5f4a4e34d5c9" dependencies = [ "jiff", "k8s-openapi", @@ -3248,7 +3268,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.119.0#ed84d456ff0b8324db3d7444659cb207564be8c9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Ffrom_docs#5d7902142fe5c7ab0a820117a80b5f4a4e34d5c9" dependencies = [ "axum", "clap", @@ -3290,12 +3310,13 @@ dependencies = [ "strum", "tokio", "tracing", + "url", ] [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.119.0#ed84d456ff0b8324db3d7444659cb207564be8c9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Ffrom_docs#5d7902142fe5c7ab0a820117a80b5f4a4e34d5c9" dependencies = [ "kube", "schemars", @@ -3309,10 +3330,10 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.119.0#ed84d456ff0b8324db3d7444659cb207564be8c9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Ffrom_docs#5d7902142fe5c7ab0a820117a80b5f4a4e34d5c9" dependencies = [ "convert_case", - "darling 0.24.0", + "darling 0.24.1", "indoc", "itertools 0.15.0", "k8s-openapi", @@ -3320,13 +3341,13 @@ dependencies = [ "kube", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.119.0#ed84d456ff0b8324db3d7444659cb207564be8c9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Ffrom_docs#5d7902142fe5c7ab0a820117a80b5f4a4e34d5c9" dependencies = [ "arc-swap", "async-trait", @@ -3339,7 +3360,7 @@ dependencies = [ "kube", "opentelemetry", "opentelemetry-semantic-conventions", - "rand 0.10.2", + "rand 0.10.3", "serde", "serde_json", "snafu 0.9.2", @@ -3349,7 +3370,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower", - "tower-http 0.7.0", + "tower-http 0.7.1", "tracing", "tracing-opentelemetry", "x509-cert", @@ -3418,9 +3439,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -3438,13 +3459,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -3458,11 +3479,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -3478,13 +3499,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] @@ -3559,9 +3580,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" dependencies = [ "bytes", "libc", @@ -3582,14 +3603,14 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -3632,9 +3653,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.15+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" dependencies = [ "indexmap", "toml_datetime", @@ -3726,7 +3747,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-util", "http", @@ -3742,11 +3763,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "http", "http-body", @@ -3789,7 +3810,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tracing-subscriber", ] @@ -3893,9 +3914,9 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-segmentation" @@ -3954,9 +3975,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.1" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" +checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" dependencies = [ "js-sys", "wasm-bindgen", @@ -3992,9 +4013,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -4016,9 +4037,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -4029,19 +4050,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4049,31 +4071,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -4302,30 +4324,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "6400fc4bb7426f6faac3fb07566122bb18622e3022bd2c36426a92ed489b1dbe" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "99c27cc7525bad4df59a45e073c45b2c2aa820efb588234cc23a547ec69a34db" dependencies = [ "proc-macro2", "quote", @@ -4343,13 +4365,13 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", "synstructure", ] @@ -4397,15 +4419,21 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.6", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 525f2e4a5..d17e72383 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ clap = "4.6" const_format = "0.2" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" -rstest = "0.26" +rstest = "0.27" semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -28,7 +28,8 @@ snafu = "0.9" strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.53", features = ["full"] } tracing = "0.1" +url = "2.5" [patch."https://github.com/stackabletech/operator-rs.git"] -# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main"} +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/from_docs" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } diff --git a/extra/crds.yaml b/extra/crds.yaml index 901063b68..c964be412 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -4329,3 +4329,867 @@ spec: type: object served: true storage: true + - name: v1alpha2 + schema: + openAPIV3Schema: + description: The TrinoCatalog resource can be used to define catalogs in Kubernetes objects. + properties: + spec: + description: |- + The TrinoCatalog resource can be used to define catalogs in Kubernetes objects. + Read more about it in the [Trino operator concept docs](https://docs.stackable.tech/home/nightly/trino/concepts) + and the [Trino operator usage guide](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/). + The documentation also contains a list of all the supported backends. + properties: + configOverrides: + additionalProperties: + type: string + default: {} + description: |- + The `configOverrides` allow overriding arbitrary Trino settings. + For example, for Hive you could add `hive.metastore.username: trino`. + type: object + connector: + description: The `connector` defines which connector is used. + oneOf: + - required: + - blackHole + - required: + - deltaLake + - required: + - googleSheet + - required: + - generic + - required: + - hive + - required: + - iceberg + - required: + - postgresql + - required: + - tpcds + - required: + - tpch + properties: + blackHole: + description: A [Black Hole](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/black-hole) connector. + type: object + deltaLake: + description: An [Delta Lake](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/delta-lake) connector. + properties: + hdfs: + description: |- + Connection to an HDFS cluster. + Please make sure that the underlying Hive metastore also has access to the HDFS. + nullable: true + properties: + configMap: + description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) providing information about the HDFS cluster. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - configMap + type: object + metastore: + description: Mandatory connection to a Hive Metastore, which will be used as a storage for metadata. + properties: + configMap: + description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) providing information about the Hive metastore. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - configMap + type: object + s3: + description: |- + Connection to an S3 store. + Please make sure that the underlying Hive metastore also has access to the S3 store. + Learn more about S3 configuration in the [S3 concept docs](https://docs.stackable.tech/home/nightly/concepts/s3). + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted + description: |- + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted + type: string + credentials: + description: |- + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) providing the requested secrets.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + type: object + reference: + type: string + type: object + required: + - metastore + type: object + generic: + description: A [generic](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/generic) connector. + properties: + connectorName: + description: |- + Name of the Trino connector. + Will be passed to `connector.name`. + type: string + properties: + additionalProperties: + oneOf: + - required: + - value + - required: + - valueFromSecret + - required: + - valueFromConfigMap + properties: + value: + type: string + valueFromConfigMap: + description: Selects a key from a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + - name + type: object + valueFromSecret: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + - name + type: object + type: object + default: {} + description: |- + A map of properties to put in the connector configuration file. + They can be specified either as a raw value or be read from a Secret or ConfigMap. + type: object + required: + - connectorName + type: object + googleSheet: + description: A [Google sheets](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/google-sheets) connector. + properties: + cache: + description: |- + Cache the contents of sheets. + This is used to reduce Google Sheets API usage and latency. + nullable: true + properties: + sheetsDataExpireAfterWrite: + description: How long to cache spreadsheet data or metadata, defaults to `5m`. + nullable: true + type: string + sheetsDataMaxCacheSize: + description: Maximum number of spreadsheets to cache, defaults to 1000. + nullable: true + type: string + type: object + credentialsSecret: + description: |- + The Secret containing the Google API JSON key file. + The key used from the Secret is `credentials`. + type: string + metadataSheetId: + description: Sheet ID of the spreadsheet, that contains the table mapping. + type: string + required: + - credentialsSecret + - metadataSheetId + type: object + hive: + description: An [Apache Hive](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/hive) connector. + properties: + hdfs: + description: |- + Connection to an HDFS cluster. + Please make sure that the underlying Hive metastore also has access to the HDFS. + nullable: true + properties: + configMap: + description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) providing information about the HDFS cluster. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - configMap + type: object + metastore: + description: Mandatory connection to a Hive Metastore, which will be used as a storage for metadata. + properties: + configMap: + description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) providing information about the Hive metastore. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - configMap + type: object + s3: + description: |- + Connection to an S3 store. + Please make sure that the underlying Hive metastore also has access to the S3 store. + Learn more about S3 configuration in the [S3 concept docs](https://docs.stackable.tech/home/nightly/concepts/s3). + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted + description: |- + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted + type: string + credentials: + description: |- + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) providing the requested secrets.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + type: object + reference: + type: string + type: object + required: + - metastore + type: object + iceberg: + description: An [Apache Iceberg](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/iceberg) connector. + properties: + catalog: + description: |- + Connection to a metadata catalog, which will be used as a storage for metadata. + + We support the following backends: + + * REST catalog + * Hive metastore + * User provided + + Details can be found on the corresponding documentation + oneOf: + - required: + - rest + - required: + - hiveMetastore + - required: + - userProvided + properties: + hiveMetastore: + description: Use a Hive metastore to store metadata. + properties: + configMap: + description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) providing information about the Hive metastore. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - configMap + type: object + rest: + description: (Recommended) use a REST catalog to store metadata. + properties: + security: + default: + none: {} + description: How to authenticate against the REST catalog. + oneOf: + - required: + - none + - required: + - oAuth2 + properties: + none: + description: Don't authenticate against the REST catalog (chosen by default). + type: object + oAuth2: + description: |- + Use OAuth2 to authenticate against the REST catalog. + + Note that we only support configuring a subset of Trino's properties, you might need to use + `configOverrides` to be able to set all [available properties](https://trino.io/docs/current/object-storage/metastores.html#iceberg-specific-metastores). + properties: + credential: + description: The credential to present to the REST catalog. + oneOf: + - required: + - tokenSecretName + - required: + - credentialSecretName + properties: + credentialSecretName: + description: The Secret needs to contain the `username` and `password` keys. + type: string + tokenSecretName: + description: |- + Authenticate using a bearer token. + + The Secret needs to contain the `token` key. + type: string + type: object + serverUri: + description: The endpoint to retrieve access token from OAuth2 Server. + format: uri + type: string + required: + - credential + - serverUri + type: object + type: object + uri: + description: URL of the rest catalog server. + format: uri + type: string + required: + - uri + type: object + userProvided: + description: |- + The operator doesn't configure any catalog, the user needs to do that, + e.g. using `configOverrides`. + type: object + type: object + hdfs: + description: |- + Connection to an HDFS cluster. + Please make sure that the underlying Hive metastore also has access to the HDFS. + nullable: true + properties: + configMap: + description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) providing information about the HDFS cluster. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - configMap + type: object + s3: + description: |- + Connection to an S3 store. + Please make sure that the underlying Hive metastore also has access to the S3 store. + Learn more about S3 configuration in the [S3 concept docs](https://docs.stackable.tech/home/nightly/concepts/s3). + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted + description: |- + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted + type: string + credentials: + description: |- + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) providing the requested secrets.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + type: object + reference: + type: string + type: object + required: + - catalog + type: object + postgresql: + description: An [PostgreSQL](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/postgresql) connector. + properties: + credentialsSecretName: + description: |- + Name of a Secret containing the `username` and `password` keys used to authenticate + against the PostgreSQL server. + type: string + database: + description: Name of the database (schema) to connect to. + type: string + host: + description: Hostname or IP address of the PostgreSQL server. + type: string + parameters: + additionalProperties: + type: string + default: {} + description: |- + Additional map of JDBC connection parameters to append to the connection URL. The given + `HashMap` will be converted to query parameters in the form of + `?param1=value1¶m2=value2`. + type: object + port: + default: 5432 + description: Port the PostgreSQL server is listening on. Defaults to `5432`. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - credentialsSecretName + - database + - host + type: object + tpcds: + description: A [TPC-DS](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/tpcds) connector. + type: object + tpch: + description: A [TPC-H](https://docs.stackable.tech/home/nightly/trino/usage-guide/catalogs/tpch) connector. + type: object + type: object + experimentalConfigRemovals: + default: [] + description: |- + List of config properties which should be removed. + + This is helpful, because Trino fails to start in case you have any unused config + properties. The removals are executed after the `configOverrides`. + + This field is experimental, and might be replaced by a more generic mechanism to edit config properties + items: + type: string + type: array + name: + default: + inferred: + replaceHyphensWithUnderscores: false + description: The name of the catalog + oneOf: + - required: + - inferred + properties: + inferred: + description: |- + Infer the catalog name from the `.metadata.name` of the TrinoCatalog resource. + + This ensures that no catalog names clash, as there can only be one TrinoCatalog with a + given name. + properties: + replaceHyphensWithUnderscores: + default: false + description: |- + Whether hyphens (`-`) in the name of the catalog should be replaced by underscores (`_`). + + This is recommended because Kubernetes only allows `a-z` and `-`, while Trino + requires quoting for catalogs containing `-` characters. This mechanism allows + you to use valid Kubernetes names, but keeps the convenience of using `_` in + catalog names. + type: boolean + type: object + type: object + required: + - connector + type: object + required: + - spec + title: TrinoCatalog + type: object + served: true + storage: false diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index bd8892701..cc22479e6 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -32,6 +32,7 @@ snafu.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true +url.workspace = true [dev-dependencies] rstest.workspace = true diff --git a/rust/operator-binary/src/catalog/commons.rs b/rust/operator-binary/src/catalog/commons.rs index da6dbfee8..d4955c113 100644 --- a/rust/operator-binary/src/catalog/commons.rs +++ b/rust/operator-binary/src/catalog/commons.rs @@ -23,13 +23,13 @@ use crate::{ CONFIG_DIR_NAME, catalog::{ TrinoCatalogName, - commons::{HdfsConnection, MetastoreConnection}, + commons::{HdfsConnection, HiveMetastoreConnection}, }, }, }; #[async_trait] -impl ExtendCatalogConfig for MetastoreConnection { +impl ExtendCatalogConfig for HiveMetastoreConnection { async fn extend_catalog_config( &self, catalog_config: &mut CatalogConfig, diff --git a/rust/operator-binary/src/catalog/config.rs b/rust/operator-binary/src/catalog/config.rs index 851620a3a..2ca97bc78 100644 --- a/rust/operator-binary/src/catalog/config.rs +++ b/rust/operator-binary/src/catalog/config.rs @@ -10,7 +10,10 @@ use stackable_operator::{ use crate::{ catalog::{FromTrinoCatalogError, ToCatalogConfig}, - crd::catalog::{TrinoCatalogConnector, TrinoCatalogName, v1alpha1}, + crd::catalog::{ + TrinoCatalogName, + v1alpha2::{self, TrinoCatalogConnector}, + }, }; #[derive(Clone, Debug)] @@ -107,7 +110,7 @@ impl CatalogConfig { pub async fn from_catalog( catalog_name: &TrinoCatalogName, - catalog: &v1alpha1::TrinoCatalog, + catalog: &v1alpha2::TrinoCatalog, client: &Client, catalog_namespace: &NamespaceName, ) -> Result { @@ -144,7 +147,7 @@ impl CatalogConfig { } } -fn calculate_env_name(catalog_name: &TrinoCatalogName, property: impl Into) -> String { +pub fn calculate_env_name(catalog_name: &TrinoCatalogName, property: impl Into) -> String { let catalog = catalog_name.to_string().replace(['.', '-'], "_"); let property = property.into().replace(['.', '-'], "_"); format!("CATALOG_{catalog}_{property}").to_uppercase() diff --git a/rust/operator-binary/src/catalog/iceberg.rs b/rust/operator-binary/src/catalog/iceberg.rs index 2b18826dc..937c81044 100644 --- a/rust/operator-binary/src/catalog/iceberg.rs +++ b/rust/operator-binary/src/catalog/iceberg.rs @@ -1,9 +1,23 @@ use async_trait::async_trait; -use stackable_operator::{client::Client, v2::types::kubernetes::NamespaceName}; +use stackable_operator::{ + client::Client, + k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector}, + v2::types::kubernetes::NamespaceName, +}; use crate::{ - catalog::{ExtendCatalogConfig, FromTrinoCatalogError, ToCatalogConfig, config::CatalogConfig}, - crd::catalog::{TrinoCatalogName, iceberg::IcebergConnector}, + catalog::{ + ExtendCatalogConfig, FromTrinoCatalogError, ToCatalogConfig, + config::{CatalogConfig, calculate_env_name}, + }, + crd::catalog::{ + TrinoCatalogName, + iceberg::{ + IcebergCatalogConnection, IcebergRestCatalogConnection, + IcebergRestCatalogOAuthCredential, IcebergRestCatalogSecurity, + }, + v1alpha2::IcebergConnector, + }, }; pub const CONNECTOR_NAME: &str = "iceberg"; @@ -25,10 +39,20 @@ impl ToCatalogConfig for IcebergConnector { // See https://trino.io/docs/current/connector/iceberg.html config.add_property("iceberg.security", "allow-all"); - if let Some(metastore) = &self.metastore { - metastore - .extend_catalog_config(&mut config, catalog_name, catalog_namespace, client) - .await?; + match &self.catalog { + IcebergCatalogConnection::Rest(iceberg_rest_catalog_connection) => { + iceberg_rest_catalog_connection + .extend_catalog_config(&mut config, catalog_name, catalog_namespace, client) + .await?; + } + IcebergCatalogConnection::HiveMetastore(hive_metastore_connection) => { + hive_metastore_connection + .extend_catalog_config(&mut config, catalog_name, catalog_namespace, client) + .await?; + } + IcebergCatalogConnection::UserProvided {} => { + // Nothing to do, the user will set it up + } } if let Some(ref s3) = self.s3 { @@ -44,3 +68,86 @@ impl ToCatalogConfig for IcebergConnector { Ok(config) } } + +#[async_trait] +impl ExtendCatalogConfig for IcebergRestCatalogConnection { + async fn extend_catalog_config( + &self, + catalog_config: &mut CatalogConfig, + catalog_name: &TrinoCatalogName, + _catalog_namespace: &NamespaceName, + _client: &Client, + ) -> Result<(), FromTrinoCatalogError> { + catalog_config.add_property("iceberg.catalog.type", "rest"); + catalog_config.add_property("iceberg.rest-catalog.uri", self.uri.as_str()); + + // We explicitly use a match here to catch further additions + match &self.security { + IcebergRestCatalogSecurity::None {} => { + catalog_config.add_property("iceberg.rest-catalog.security", "NONE"); + } + IcebergRestCatalogSecurity::OAuth2 { + server_uri, + credential, + } => { + catalog_config.add_property("iceberg.rest-catalog.security", "OAUTH2"); + catalog_config.add_property( + "iceberg.rest-catalog.oauth2.server-uri", + server_uri.as_str(), + ); + match credential { + IcebergRestCatalogOAuthCredential::TokenSecretName(secret_name) => { + // We can't use `add_env_property_from_secret`, as we need to concatenate + // user and password in the Trino configuration. So instead we come up with + // our own envs and bind them. + let property = "iceberg.rest-catalog.oauth2.credential"; + let base_env_name = calculate_env_name(catalog_name, property); + let username_env_name = format!("{base_env_name}_USERNAME"); + let password_env_name = format!("{base_env_name}_PASSWORD"); + catalog_config.add_property( + property, + format!("${{ENV:{username_env_name}}}:${{ENV:{password_env_name}}}"), + ); + + catalog_config.env_bindings.push(EnvVar { + name: username_env_name, + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: secret_name.to_owned(), + key: "username".to_owned(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }); + catalog_config.env_bindings.push(EnvVar { + name: password_env_name, + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: secret_name.to_owned(), + key: "password".to_owned(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }); + } + IcebergRestCatalogOAuthCredential::CredentialSecretName(secret_name) => { + catalog_config.add_env_property_from_secret( + "iceberg.rest-catalog.oauth2.token", + SecretKeySelector { + name: secret_name.to_owned(), + key: "token".to_owned(), + ..Default::default() + }, + ) + } + } + } + } + + Ok(()) + } +} diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index fe7af645e..78327964b 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -49,7 +49,7 @@ pub enum Error { #[snafu(display("failed to parse {catalog}"))] ParseCatalog { source: FromTrinoCatalogError, - catalog: ObjectRef, + catalog: ObjectRef, }, #[snafu(display("failed to configure fault tolerant execution"))] @@ -77,7 +77,7 @@ type Result = std::result::Result; /// Kubernetes objects referenced from the TrinoCluster spec, fetched from the cluster. pub struct DereferencedObjects { pub resolved_authentication_classes: Vec, - pub catalog_definitions: Vec, + pub catalog_definitions: Vec, pub catalogs: Vec, pub trino_opa_config: Option, pub resolved_fte_config: Option, @@ -97,7 +97,7 @@ pub async fn dereference( .context(AuthenticationClassRetrievalSnafu)?; let catalog_definitions = client - .list_with_label_selector::( + .list_with_label_selector::( namespace.as_ref(), &trino.spec.cluster_config.catalog_label_selector, ) @@ -168,12 +168,12 @@ pub async fn dereference( /// Determines the Trino catalog name based on user settings and the Kubernetes TrinoCatalog object /// name. fn determine_catalog_name( - catalog_name_spec: &catalog::v1alpha1::TrinoCatalogNameSpec, + catalog_name_spec: &catalog::v1alpha2::TrinoCatalogNameSpec, catalog_object_name: &str, ) -> Result { // A `match` is used because we might support other ways of naming (e.g. custom) later. match catalog_name_spec { - catalog::v1alpha1::TrinoCatalogNameSpec::Inferred { + catalog::v1alpha2::TrinoCatalogNameSpec::Inferred { replace_hyphens_with_underscores, } => { let mut catalog_name = catalog_object_name.to_owned(); @@ -194,7 +194,7 @@ mod tests { fn determine_catalog_name_replaces_hyphens() { let inferred = |replace_hyphens_with_underscores| { determine_catalog_name( - &catalog::v1alpha1::TrinoCatalogNameSpec::Inferred { + &catalog::v1alpha2::TrinoCatalogNameSpec::Inferred { replace_hyphens_with_underscores, }, "my-postgres", diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index f6dc77dd3..1c6b78f81 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -375,7 +375,7 @@ pub(crate) fn merged_role_group_config( trino: &v1alpha1::TrinoCluster, trino_role: &TrinoRole, role_group: &str, - trino_catalogs: &[crate::crd::catalog::v1alpha1::TrinoCatalog], + trino_catalogs: &[crate::crd::catalog::v1alpha2::TrinoCatalog], ) -> TrinoRoleGroupConfig { // The shared test clusters do not enable the Vector agent, so no aggregator ConfigMap name is // required here. diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index fdcb9f00f..c5484c13f 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -7,75 +7,86 @@ use stackable_operator::{ use crate::crd::{ APP_NAME, TrinoRole, - catalog::{self, TrinoCatalogConnector}, + catalog::{ + self, + commons::HiveMetastoreConnection, + iceberg::IcebergCatalogConnection, + v1alpha2::{IcebergConnector, TrinoCatalogConnector}, + }, v1alpha1, }; pub fn get_affinity( cluster_name: &str, role: &TrinoRole, - trino_catalogs: &[catalog::v1alpha1::TrinoCatalog], + trino_catalogs: &[catalog::v1alpha2::TrinoCatalog], opa_config: Option<&v1alpha1::TrinoAuthorizationOpaConfig>, ) -> StackableAffinityFragment { let affinity_between_cluster_pods = affinity_between_cluster_pods(APP_NAME, cluster_name, 20); let mut affinities = vec![affinity_between_cluster_pods]; - let additional_affinities: Vec<_> = match role { - TrinoRole::Coordinator => trino_catalogs - .iter() - .filter_map(|catalog| match &catalog.spec.connector { - TrinoCatalogConnector::Hive(hive) => Some(&hive.metastore.config_map), - TrinoCatalogConnector::Iceberg(iceberg) => iceberg - .metastore - .as_ref() - .map(|metastore| &metastore.config_map), - TrinoCatalogConnector::DeltaLake(delta_lake) => { - Some(&delta_lake.metastore.config_map) - } - TrinoCatalogConnector::BlackHole(_) - | TrinoCatalogConnector::Generic(_) - | TrinoCatalogConnector::GoogleSheet(_) - | TrinoCatalogConnector::Postgresql(_) - | TrinoCatalogConnector::Tpcds(_) - | TrinoCatalogConnector::Tpch(_) => None, - }) - .map(|hive_cluster_name| { - affinity_between_role_pods( - "hive", - hive_cluster_name.as_ref(), // The discovery cm has the same name as the HiveCluster itself - "metastore", - 50, - ) - }) - .collect(), - TrinoRole::Worker => trino_catalogs - .iter() - .filter_map(|catalog| match &catalog.spec.connector { - TrinoCatalogConnector::Hive(hive) => { - hive.hdfs.as_ref().map(|hdfs| &hdfs.config_map) - } - TrinoCatalogConnector::Iceberg(iceberg) => { - iceberg.hdfs.as_ref().map(|hdfs| &hdfs.config_map) - } - TrinoCatalogConnector::DeltaLake(delta_lake) => { - delta_lake.hdfs.as_ref().map(|hdfs| &hdfs.config_map) - } - TrinoCatalogConnector::BlackHole(_) - | TrinoCatalogConnector::Generic(_) - | TrinoCatalogConnector::GoogleSheet(_) - | TrinoCatalogConnector::Postgresql(_) - | TrinoCatalogConnector::Tpcds(_) - | TrinoCatalogConnector::Tpch(_) => None, - }) - .map(|hdfs_cluster_name| { - affinity_between_role_pods( - "hdfs", - hdfs_cluster_name.as_ref(), // The discovery cm has the same name as the HdfsCluster itself - "datanode", - 50, - ) - }) - .collect(), - }; + let additional_affinities: Vec<_> = + match role { + TrinoRole::Coordinator => trino_catalogs + .iter() + .filter_map(|catalog| match &catalog.spec.connector { + TrinoCatalogConnector::Hive(hive) => Some(&hive.metastore.config_map), + TrinoCatalogConnector::Iceberg(IcebergConnector { + catalog: + IcebergCatalogConnection::HiveMetastore(HiveMetastoreConnection { + config_map, + }), + .. + }) => Some(config_map), + // No Hive metastore is used + TrinoCatalogConnector::Iceberg(_) => None, + TrinoCatalogConnector::DeltaLake(delta_lake) => { + Some(&delta_lake.metastore.config_map) + } + TrinoCatalogConnector::BlackHole(_) + | TrinoCatalogConnector::Generic(_) + | TrinoCatalogConnector::GoogleSheet(_) + | TrinoCatalogConnector::Postgresql(_) + | TrinoCatalogConnector::Tpcds(_) + | TrinoCatalogConnector::Tpch(_) => None, + }) + .map(|hive_cluster_name| { + affinity_between_role_pods( + "hive", + hive_cluster_name.as_ref(), // The discovery cm has the same name as the HiveCluster itself + "metastore", + 50, + ) + }) + .collect(), + TrinoRole::Worker => trino_catalogs + .iter() + .filter_map(|catalog| match &catalog.spec.connector { + TrinoCatalogConnector::Hive(hive) => { + hive.hdfs.as_ref().map(|hdfs| &hdfs.config_map) + } + TrinoCatalogConnector::Iceberg(iceberg) => { + iceberg.hdfs.as_ref().map(|hdfs| &hdfs.config_map) + } + TrinoCatalogConnector::DeltaLake(delta_lake) => { + delta_lake.hdfs.as_ref().map(|hdfs| &hdfs.config_map) + } + TrinoCatalogConnector::BlackHole(_) + | TrinoCatalogConnector::Generic(_) + | TrinoCatalogConnector::GoogleSheet(_) + | TrinoCatalogConnector::Postgresql(_) + | TrinoCatalogConnector::Tpcds(_) + | TrinoCatalogConnector::Tpch(_) => None, + }) + .map(|hdfs_cluster_name| { + affinity_between_role_pods( + "hdfs", + hdfs_cluster_name.as_ref(), // The discovery cm has the same name as the HdfsCluster itself + "datanode", + 50, + ) + }) + .collect(), + }; affinities.extend(additional_affinities); // Only the coordinator talks to OPA (it does the authorization checks for the whole cluster), @@ -261,7 +272,7 @@ mod tests { hdfs: configMap: simple-hdfs "#; - let hive_catalog_1: catalog::v1alpha1::TrinoCatalog = + let hive_catalog_1: catalog::v1alpha2::TrinoCatalog = yaml_from_str_singleton_map(input).expect("illegal test input"); let input = r#" @@ -275,7 +286,7 @@ mod tests { connector: tpch: {} "#; - let tpch_catalog: catalog::v1alpha1::TrinoCatalog = + let tpch_catalog: catalog::v1alpha2::TrinoCatalog = yaml_from_str_singleton_map(input).expect("illegal test input"); let input = r#" @@ -293,7 +304,7 @@ mod tests { s3: reference: minio "#; - let hive_catalog_2: catalog::v1alpha1::TrinoCatalog = + let hive_catalog_2: catalog::v1alpha2::TrinoCatalog = yaml_from_str_singleton_map(input).expect("illegal test input"); let merged_config = crate::controller::validate::merged_role_group_config( diff --git a/rust/operator-binary/src/crd/catalog/commons.rs b/rust/operator-binary/src/crd/catalog/commons.rs index a4fe7c2ef..70e6a4aa1 100644 --- a/rust/operator-binary/src/crd/catalog/commons.rs +++ b/rust/operator-binary/src/crd/catalog/commons.rs @@ -6,7 +6,7 @@ use stackable_operator::{ #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct MetastoreConnection { +pub struct HiveMetastoreConnection { /// Name of the [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) providing information about the Hive metastore. pub config_map: ConfigMapName, } diff --git a/rust/operator-binary/src/crd/catalog/delta_lake.rs b/rust/operator-binary/src/crd/catalog/delta_lake.rs index 1d200c0a6..7d915ceae 100644 --- a/rust/operator-binary/src/crd/catalog/delta_lake.rs +++ b/rust/operator-binary/src/crd/catalog/delta_lake.rs @@ -4,7 +4,7 @@ use stackable_operator::{ schemars::{self, JsonSchema}, }; -use super::commons::{HdfsConnection, MetastoreConnection}; +use super::commons::{HdfsConnection, HiveMetastoreConnection}; // This struct is similar to [`super::hive::HiveConnector`], but we do not `#[serde(flatten)]` it here, to avoid changing // stuff there and missing that these settings don't apply to other connectors (such as Iceberg or Delta Lake). @@ -12,7 +12,7 @@ use super::commons::{HdfsConnection, MetastoreConnection}; #[serde(rename_all = "camelCase")] pub struct DeltaLakeConnector { /// Mandatory connection to a Hive Metastore, which will be used as a storage for metadata. - pub metastore: MetastoreConnection, + pub metastore: HiveMetastoreConnection, /// Connection to an S3 store. /// Please make sure that the underlying Hive metastore also has access to the S3 store. diff --git a/rust/operator-binary/src/crd/catalog/generic.rs b/rust/operator-binary/src/crd/catalog/generic.rs index a83e6c78e..ea6d8ac99 100644 --- a/rust/operator-binary/src/crd/catalog/generic.rs +++ b/rust/operator-binary/src/crd/catalog/generic.rs @@ -38,7 +38,7 @@ pub enum Property { #[cfg(test)] mod tests { use super::*; - use crate::crd::catalog::{TrinoCatalogConnector, v1alpha1}; + use crate::crd::catalog::v1alpha1; #[test] fn test_cr_parsing() { @@ -71,7 +71,7 @@ mod tests { assert_eq!( catalog.spec.connector, - TrinoCatalogConnector::Generic(GenericConnector { + v1alpha1::TrinoCatalogConnector::Generic(GenericConnector { connector_name: "postgresql".to_string(), properties: BTreeMap::from([ ( diff --git a/rust/operator-binary/src/crd/catalog/hive.rs b/rust/operator-binary/src/crd/catalog/hive.rs index edd5cf717..dcc9c28a4 100644 --- a/rust/operator-binary/src/crd/catalog/hive.rs +++ b/rust/operator-binary/src/crd/catalog/hive.rs @@ -4,13 +4,13 @@ use stackable_operator::{ schemars::{self, JsonSchema}, }; -use super::commons::{HdfsConnection, MetastoreConnection}; +use super::commons::{HdfsConnection, HiveMetastoreConnection}; #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HiveConnector { /// Mandatory connection to a Hive Metastore, which will be used as a storage for metadata. - pub metastore: MetastoreConnection, + pub metastore: HiveMetastoreConnection, /// Connection to an S3 store. /// Please make sure that the underlying Hive metastore also has access to the S3 store. diff --git a/rust/operator-binary/src/crd/catalog/iceberg.rs b/rust/operator-binary/src/crd/catalog/iceberg.rs index 1b0425aef..715e72cff 100644 --- a/rust/operator-binary/src/crd/catalog/iceberg.rs +++ b/rust/operator-binary/src/crd/catalog/iceberg.rs @@ -1,28 +1,70 @@ use serde::{Deserialize, Serialize}; -use stackable_operator::{ - crd::s3, - schemars::{self, JsonSchema}, -}; +use stackable_operator::schemars::{self, JsonSchema}; +use url::Url; -use super::commons::{HdfsConnection, MetastoreConnection}; +use super::commons::HiveMetastoreConnection; -// This struct is similar to [`super::hive::HiveConnector`], but we do not `#[serde(flatten)]` it here, to avoid changing -// stuff there and missing that these settings don't apply to other connectors (such as Iceberg or Delta Lake). #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct IcebergConnector { - /// Optional connection to a Hive Metastore, which will be used as a storage for metadata. +pub enum IcebergCatalogConnection { + /// (Recommended) use a REST catalog to store metadata. + Rest(IcebergRestCatalogConnection), + + /// Use a Hive metastore to store metadata. + HiveMetastore(HiveMetastoreConnection), + + /// The operator doesn't configure any catalog, the user needs to do that, + /// e.g. using `configOverrides`. + UserProvided {}, +} + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct IcebergRestCatalogConnection { + /// URL of the rest catalog server. + pub uri: Url, + + /// How to authenticate against the REST catalog. + #[serde(default)] + pub security: IcebergRestCatalogSecurity, +} + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum IcebergRestCatalogSecurity { + /// Don't authenticate against the REST catalog (chosen by default). + None {}, + + /// Use OAuth2 to authenticate against the REST catalog. + /// + /// Note that we only support configuring a subset of Trino's properties, you might need to use + /// `configOverrides` to be able to set all [available properties](https://trino.io/docs/current/object-storage/metastores.html#iceberg-specific-metastores). + #[serde(rename_all = "camelCase")] + OAuth2 { + /// The endpoint to retrieve access token from OAuth2 Server. + server_uri: Url, + + /// The credential to present to the REST catalog. + credential: IcebergRestCatalogOAuthCredential, + }, +} + +impl Default for IcebergRestCatalogSecurity { + fn default() -> Self { + Self::None {} + } +} + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum IcebergRestCatalogOAuthCredential { + /// Authenticate using a bearer token. + /// + /// The Secret needs to contain the `token` key. + TokenSecretName(String), + + // The credential to exchange for a token in the OAuth2 client credentials flow with the server. /// - /// The connection is optional, as Iceberg also supports other catalogs, such as a REST catalog, - /// which (currently) can only be added using configOverrides. - pub metastore: Option, - - /// Connection to an S3 store. - /// Please make sure that the underlying Hive metastore also has access to the S3 store. - /// Learn more about S3 configuration in the [S3 concept docs](DOCS_BASE_URL_PLACEHOLDER/concepts/s3). - pub s3: Option, - - /// Connection to an HDFS cluster. - /// Please make sure that the underlying Hive metastore also has access to the HDFS. - pub hdfs: Option, + /// The Secret needs to contain the `username` and `password` keys. + CredentialSecretName(String), } diff --git a/rust/operator-binary/src/crd/catalog/mod.rs b/rust/operator-binary/src/crd/catalog/mod.rs index e65004257..6a3913ad1 100644 --- a/rust/operator-binary/src/crd/catalog/mod.rs +++ b/rust/operator-binary/src/crd/catalog/mod.rs @@ -15,7 +15,6 @@ use black_hole::BlackHoleConnector; use generic::GenericConnector; use google_sheet::GoogleSheetConnector; use hive::HiveConnector; -use iceberg::IcebergConnector; use serde::{Deserialize, Serialize}; use stackable_operator::{ attributed_string_type, @@ -26,19 +25,23 @@ use stackable_operator::{ use tpcds::TpcdsConnector; use tpch::TpchConnector; -use self::delta_lake::DeltaLakeConnector; -use crate::crd::catalog::postgresql::PostgresqlConnector; +use crate::crd::catalog::{ + commons::{HdfsConnection, HiveMetastoreConnection}, + delta_lake::DeltaLakeConnector, + iceberg::IcebergCatalogConnection, + postgresql::PostgresqlConnector, +}; #[versioned( version(name = "v1alpha1"), + version(name = "v1alpha2"), crates( kube_core = "stackable_operator::kube::core", kube_client = "stackable_operator::kube::client", k8s_openapi = "stackable_operator::k8s_openapi", schemars = "stackable_operator::schemars", versioned = "stackable_operator::versioned", - ), - skip(from) + ) )] pub mod versioned { /// The TrinoCatalog resource can be used to define catalogs in Kubernetes objects. @@ -97,6 +100,74 @@ pub mod versioned { replace_hyphens_with_underscores: bool, }, } + + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub enum TrinoCatalogConnector { + /// A [Black Hole](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/black-hole) connector. + BlackHole(BlackHoleConnector), + + /// An [Delta Lake](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/delta-lake) connector. + DeltaLake(DeltaLakeConnector), + + /// A [Google sheets](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/google-sheets) connector. + GoogleSheet(GoogleSheetConnector), + + /// A [generic](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/generic) connector. + Generic(GenericConnector), + + /// An [Apache Hive](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/hive) connector. + Hive(HiveConnector), + + /// An [Apache Iceberg](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/iceberg) connector. + Iceberg(IcebergConnector), + + /// An [PostgreSQL](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/postgresql) connector. + Postgresql(PostgresqlConnector), + + /// A [TPC-DS](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/tpcds) connector. + Tpcds(TpcdsConnector), + + /// A [TPC-H](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/tpch) connector. + Tpch(TpchConnector), + } + + // Note that this struct needs to live here (instead of in `iceberg.rs`), so that it's part of + // the same `versioned` module as the struct that reference it. + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct IcebergConnector { + /// Connection to a metadata catalog, which will be used as a storage for metadata. + /// + /// We support the following backends: + /// + /// * REST catalog + /// * Hive metastore + /// * User provided + /// + /// Details can be found on the corresponding documentation + #[versioned(changed( + since = "v1alpha2", + from_name = "metastore", + from_type = "Option", + from_docs = r#" + Optional connection to a Hive Metastore, which will be used as a storage for metadata. + + The connection is optional, as Iceberg also supports other catalogs, such as a REST catalog, + which (currently) can only be added using configOverrides. + "# + ))] + pub catalog: IcebergCatalogConnection, + + /// Connection to an S3 store. + /// Please make sure that the underlying Hive metastore also has access to the S3 store. + /// Learn more about S3 configuration in the [S3 concept docs](DOCS_BASE_URL_PLACEHOLDER/concepts/s3). + pub s3: Option, + + /// Connection to an HDFS cluster. + /// Please make sure that the underlying Hive metastore also has access to the HDFS. + pub hdfs: Option, + } } impl Default for v1alpha1::TrinoCatalogNameSpec { @@ -107,35 +178,37 @@ impl Default for v1alpha1::TrinoCatalogNameSpec { } } -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub enum TrinoCatalogConnector { - /// A [Black Hole](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/black-hole) connector. - BlackHole(BlackHoleConnector), - - /// An [Delta Lake](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/delta-lake) connector. - DeltaLake(DeltaLakeConnector), - - /// A [Google sheets](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/google-sheets) connector. - GoogleSheet(GoogleSheetConnector), - - /// A [generic](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/generic) connector. - Generic(GenericConnector), - - /// An [Apache Hive](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/hive) connector. - Hive(HiveConnector), - - /// An [Apache Iceberg](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/iceberg) connector. - Iceberg(IcebergConnector), - - /// An [PostgreSQL](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/postgresql) connector. - Postgresql(PostgresqlConnector), +impl Default for v1alpha2::TrinoCatalogNameSpec { + fn default() -> Self { + Self::Inferred { + replace_hyphens_with_underscores: false, + } + } +} - /// A [TPC-DS](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/tpcds) connector. - Tpcds(TpcdsConnector), +impl From for Option { + fn from(value: IcebergCatalogConnection) -> Self { + match value { + IcebergCatalogConnection::Rest(_iceberg_rest_catalog_connection) => { + todo!("Puh, how should we map REST to old?") + } + IcebergCatalogConnection::HiveMetastore(hive_metastore_connection) => { + Some(hive_metastore_connection) + } + // Earlier the metastore connection was optional, so that users could bring their ow + // (e.g. REST catalog, as the operator didn't support that back than). + IcebergCatalogConnection::UserProvided {} => None, + } + } +} - /// A [TPC-H](DOCS_BASE_URL_PLACEHOLDER/trino/usage-guide/catalogs/tpch) connector. - Tpch(TpchConnector), +impl From> for IcebergCatalogConnection { + fn from(value: Option) -> Self { + match value { + Some(hive_metastore_connection) => Self::HiveMetastore(hive_metastore_connection), + None => Self::UserProvided {}, + } + } } attributed_string_type! { @@ -155,7 +228,7 @@ attributed_string_type! { mod tests { use stackable_operator::versioned::test_utils::RoundtripTestData; - use super::{TrinoCatalog, TrinoCatalogVersion, v1alpha1}; + use super::{TrinoCatalog, TrinoCatalogVersion, v1alpha1, v1alpha2}; #[test] fn test_crd_generation() { @@ -223,6 +296,101 @@ mod tests { accessStyle: Path credentials: secretClass: minio-credentials + - connector: + iceberg: {} + - connector: + tpcds: {} + - name: + inferred: + replaceHyphensWithUnderscores: true + connector: + tpch: {} + "}) + .expect("Failed to parse TrinoCatalogSpec YAML") + } + } + + impl RoundtripTestData for v1alpha2::TrinoCatalogSpec { + fn roundtrip_test_data() -> Vec { + stackable_operator::utils::yaml_from_str_singleton_map(indoc::indoc! {" + - connector: + blackHole: {} + - connector: + deltaLake: + metastore: + configMap: simple-hive + s3: + inline: + host: test-minio + port: 9000 + accessStyle: Path + credentials: + secretClass: minio-credentials + - connector: + generic: + connectorName: postgresql + properties: # optional + connection-url: + value: jdbc:postgresql://example.net:5432/database + connection-user: + valueFromSecret: + name: my-postgresql-credentials-secret + key: user + connection-password: + valueFromSecret: + name: my-postgresql-credentials-secret + key: password + - connector: + googleSheet: + credentialsSecret: gsheet-credentials + metadataSheetId: 1dT4dRWo9tAKBk5GdH-a54dcizuoxOTn98X8igZcnYr8 + cache: # optional + sheetsDataMaxCacheSize: 1000 + sheetsDataExpireAfterWrite: 5m + - connector: + hive: + metastore: + configMap: simple-hive + s3: + inline: + host: test-minio + port: 9000 + accessStyle: Path + credentials: + secretClass: minio-credentials + configOverrides: + hive.metastore.username: trino + - connector: + iceberg: + catalog: + hiveMetastore: + configMap: simple-hive + s3: + inline: + host: test-minio + port: 9000 + accessStyle: Path + credentials: + secretClass: minio-credentials + - connector: + iceberg: + catalog: + rest: + uri: https://my.rest.com/iceberg + - connector: + iceberg: + catalog: + rest: + uri: https://my.secure.rest + security: + oAuth2: + serverUri: https://keycloak.default.svc.cluster.local:8443/realms/test/protocol/openid-connect/token + credential: + credentialSecretName: my-keycloak-credentials + - connector: + iceberg: + catalog: + userProvided: {} - connector: tpcds: {} - name: diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index c567fd42d..1f4032400 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -476,7 +476,7 @@ impl v1alpha1::TrinoConfig { pub(crate) fn default_config( cluster_name: &str, role: &TrinoRole, - trino_catalogs: &[catalog::v1alpha1::TrinoCatalog], + trino_catalogs: &[catalog::v1alpha2::TrinoCatalog], opa_config: Option<&v1alpha1::TrinoAuthorizationOpaConfig>, ) -> v1alpha1::TrinoConfigFragment { let (cpu_min, cpu_max, memory) = match role {