From 5318ba59a9ce4154422a4fc8f0758eaafbb6e69e Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 30 Sep 2026 14:22:11 +0200 Subject: [PATCH 1/2] feat(operator)!: Introduce v1alpha2 Role and RoleGroup This commit includes a bunch of changes: - The Role and RoleGroup types are now versioned via our versioned macro. - It introduces two new versions: v1alpha1 and v1alpha2. v1alpha1 represents the shape of these types unchanged up until now. v1alpha2 introduces the changes which made versioning these types necessary at all. - In v1alpha2 the replicas field was changed from a simple Option to a complex enum to support our planned auto-scaling mechanism. This is a breaking change for primarily two reasons: - The shape of Role and RoleGroup changed in v1alpha2. - The location of the Role, RoleGroup, and supporting code (like function and helper types) changed. The code is now located at stackable_operator::crd::role. --- .../stackable-operator/crds/DummyCluster.yaml | 50 +- .../src/cluster_resources.rs | 1 - .../src/commons/resources.rs | 5 +- .../stackable-operator/src/config/fragment.rs | 14 +- crates/stackable-operator/src/config/mod.rs | 12 +- crates/stackable-operator/src/crd/mod.rs | 20 +- .../stackable-operator/src/crd/role/java.rs | 251 ++++++ crates/stackable-operator/src/crd/role/mod.rs | 399 ++++++++++ .../stackable-operator/src/crd/role/utils.rs | 105 +++ .../src/crd/role/v1alpha2_impl.rs | 74 ++ crates/stackable-operator/src/lib.rs | 24 +- .../src/product_logging/framework.rs | 4 +- crates/stackable-operator/src/role_utils.rs | 729 ------------------ crates/stackable-operator/src/test_utils.rs | 11 + .../src/v2/jvm_argument_overrides.rs | 2 +- .../stackable-operator/src/v2/role_utils.rs | 8 +- crates/xtask/src/crd/dummy.rs | 4 +- 17 files changed, 944 insertions(+), 769 deletions(-) create mode 100644 crates/stackable-operator/src/crd/role/java.rs create mode 100644 crates/stackable-operator/src/crd/role/mod.rs create mode 100644 crates/stackable-operator/src/crd/role/utils.rs create mode 100644 crates/stackable-operator/src/crd/role/v1alpha2_impl.rs delete mode 100644 crates/stackable-operator/src/role_utils.rs diff --git a/crates/stackable-operator/crds/DummyCluster.yaml b/crates/stackable-operator/crds/DummyCluster.yaml index 0f3a96838..9c5289d68 100644 --- a/crates/stackable-operator/crds/DummyCluster.yaml +++ b/crates/stackable-operator/crds/DummyCluster.yaml @@ -418,13 +418,16 @@ spec: type: string nodes: description: |- - This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a key-value-map containing - all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable - at the role *and* roleGroup level. Everything at roleGroup level is merged on top of what is configured - on role level. There is also a second form of config, which can only be configured - at role level, the `roleConfig`. - You can learn more about this in the - [Roles and role group concept documentation](https://docs.stackable.tech/home/nightly/concepts/roles-and-role-groups). + This struct represents a role - e.g. HDFS datanodes or Trino workers. + + It has a key-value-map containing all the roleGroups that are part of this role. Additionally, there is a + `config`, which is configurable at the role **and** roleGroup level. Everything at roleGroup level is merged on + top of what is configured on role level. + + There is also a second form of config, which can only be configured at role level, the `roleConfig`. You can + learn more about this in the [Roles and role group concept documentation][1]. + + [1]: https://docs.stackable.tech/home/nightly/concepts/roles-and-role-groups nullable: true properties: cliOverrides: @@ -1744,11 +1747,34 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true replicas: - format: uint16 - maximum: 65535.0 - minimum: 0.0 - nullable: true - type: integer + oneOf: + - required: + - fixed + - required: + - managed + - required: + - custom + properties: + custom: + description: Use a auto-scaled number of replicas managed by external/custom mechanisms, e.g. HPA. + type: object + fixed: + description: Use a fixed number of replicas. + properties: + count: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + type: integer + required: + - count + type: object + managed: + description: Use a fully managed, auto-scaled number of replicas. + type: object + type: object + required: + - replicas type: object description: |- The set of role groups for this role, keyed by their name. diff --git a/crates/stackable-operator/src/cluster_resources.rs b/crates/stackable-operator/src/cluster_resources.rs index 4cd6d93e9..c82448d87 100644 --- a/crates/stackable-operator/src/cluster_resources.rs +++ b/crates/stackable-operator/src/cluster_resources.rs @@ -338,7 +338,6 @@ impl ClusterResource for Deployment { /// client::Client, /// cluster_resources::{self, ClusterResourceApplyStrategy, ClusterResources}, /// deep_merger::ObjectOverrides, -/// role_utils::Role, /// }; /// /// const APP_NAME: &str = "app"; diff --git a/crates/stackable-operator/src/commons/resources.rs b/crates/stackable-operator/src/commons/resources.rs index f03babeee..75272c02c 100644 --- a/crates/stackable-operator/src/commons/resources.rs +++ b/crates/stackable-operator/src/commons/resources.rs @@ -21,13 +21,15 @@ //! # Example //! //! ``` +//! # #[cfg(feature = "crds")] +//! # mod wrapper { //! use kube::CustomResource; //! use schemars::JsonSchema; //! use serde::{Deserialize, Serialize}; //! use stackable_operator::{ //! commons::resources::{JvmHeapLimits, PvcConfig, Resources}, //! config::fragment::Fragment, -//! role_utils::Role, +//! crd::role::v1alpha2::Role, //! }; //! //! #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -71,6 +73,7 @@ //! metadata_storage: PvcConfig, //! shared_storage: PvcConfig, //! } +//! # } //! ``` use std::{collections::BTreeMap, fmt::Debug}; diff --git a/crates/stackable-operator/src/config/fragment.rs b/crates/stackable-operator/src/config/fragment.rs index a1a89f6be..22342da18 100644 --- a/crates/stackable-operator/src/config/fragment.rs +++ b/crates/stackable-operator/src/config/fragment.rs @@ -16,8 +16,8 @@ pub use stackable_operator_derive::Fragment; use super::merge::Atomic; #[cfg(doc)] use super::merge::Merge; -#[cfg(doc)] -use crate::role_utils::{Role, RoleGroup}; +#[cfg(all(doc, feature = "crds"))] +use crate::crd::role::v1alpha2::{Role, RoleGroup}; /// Contains context used for generating validation errors /// @@ -93,13 +93,15 @@ enum ValidationProblem { /// A type that can be constructed by validating a "fragment" type. /// /// This is intended to be used together with [`Merge`], such that fragments are deserialized from multiple sources -/// (for example: the [`RoleGroup`] and [`Role`] levels of a ProductCluster object), and then validated into the type implementing +/// (for example: the [role group][role] and [role] levels of a ProductCluster object), and then validated into the type implementing /// `FromFragment`. /// -/// It is recommended to use [`RoleGroup::validate_config`] to both merge and validate product [`RoleGroup`] configurations. For other use cases, +/// It is recommended to use `RoleGroup::validate_config` to both merge and validate product [role group][role] configurations. For other use cases, /// [`validate`] can be used on the already-merged configuration. /// /// This will typically be derived using the [`Fragment`] macro, rather than implemented manually. +/// +/// [role]: crate::crd::role pub trait FromFragment: Sized { /// The fragment type of `Self`. /// @@ -206,7 +208,9 @@ impl FromFragment for PodTemplateSpec { /// Validates a [`Fragment`](`FromFragment::Fragment`), and turns it into its corresponding [`FromFragment`] type if successful. /// -/// When validating a [`RoleGroup`]'s configuration, consider using [`RoleGroup::validate_config`] instead. +/// When validating a [`RoleGroup`][role]'s configuration, consider using `RoleGroup::validate_config` instead. +/// +/// [role]: crate::crd::role pub fn validate(fragment: T::Fragment) -> Result { T::from_fragment( fragment, diff --git a/crates/stackable-operator/src/config/mod.rs b/crates/stackable-operator/src/config/mod.rs index 9c17f6048..6f4c5cfc1 100644 --- a/crates/stackable-operator/src/config/mod.rs +++ b/crates/stackable-operator/src/config/mod.rs @@ -3,8 +3,8 @@ //! # But oh god why is this monstrosity a thing? //! //! Products are complicated. They need to be supplied many kinds of configuration. -//! Some of it applies to the whole installation (Stacklet). Some of it applies only to one [role](`Role`). -//! Some of it applies only to a subset of the instances of that role (we call this a [`RoleGroup`]). +//! Some of it applies to the whole installation (Stacklet). Some of it applies only to one [role]. +//! Some of it applies only to a subset of the instances of that role (we call this a [role group][role]). //! //! We (usually) don't know at what level it makes sense to apply a given piece of configuration, but we also //! don't want to force users to repeat themselves constantly! Instead, we model the configuration as a tree: @@ -145,6 +145,7 @@ //! # Fine, how do I actually use it, then? //! //! For declarations (in CRDs): +//! //! - Apply `#[derive(Fragment)] #[fragment_attrs(derive(Merge))]` for your product configuration (and any of its nested types). //! - DON'T: `#[derive(Fragment, Merge)]` //! - Pretty much always derive deserialization and defaulting on the `Fragment`, not the validated type: @@ -155,12 +156,14 @@ //! - Define the "validated form" of your configuration: only make fields [`Option`]al if [`None`] is actually a legal value. //! //! For runtime code: -//! - Validate and merge with [`RoleGroup::validate_config`] for CRDs, otherwise [`merge`] manually and then validate with [`validate`]. +//! +//! - Validate and merge with `RoleGroup::validate_config` for CRDs, otherwise [`merge`] manually and then validate with [`validate`]. //! - Validate as soon as possible, user code should never read the contents of `Fragment`s. //! - Defaults are just another layer to be [`merge`]d. //! //! [parsing]: https://lexi-lambda.github.io/blog/2019/11/05/parse-don-t-validate/ //! [`merge`]: Merge::merge +//! [role]: crate::crd::role pub mod fragment; pub mod merge; @@ -169,6 +172,3 @@ pub mod merge; use fragment::{Fragment, FromFragment, validate}; #[cfg(doc)] use merge::{Atomic, Merge}; - -#[cfg(doc)] -use crate::role_utils::{Role, RoleGroup}; diff --git a/crates/stackable-operator/src/crd/mod.rs b/crates/stackable-operator/src/crd/mod.rs index 28f40f997..061b8eb13 100644 --- a/crates/stackable-operator/src/crd/mod.rs +++ b/crates/stackable-operator/src/crd/mod.rs @@ -4,11 +4,21 @@ use educe::Educe; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -pub mod authentication; -pub mod git_sync; -pub mod listener; -pub mod s3; -pub mod scaler; +use crate::cfg_block; + +cfg_block! { + "crds"; + + pub mod authentication; + pub mod git_sync; + pub mod listener; + pub mod s3; + pub mod scaler; +} + +// Not gated as a whole as there are other items which should not be gated, but are still related to +// role and role group. +pub mod role; /// A reference to a product cluster (for example, a `ZookeeperCluster`) /// diff --git a/crates/stackable-operator/src/crd/role/java.rs b/crates/stackable-operator/src/crd/role/java.rs new file mode 100644 index 000000000..242fc7e48 --- /dev/null +++ b/crates/stackable-operator/src/crd/role/java.rs @@ -0,0 +1,251 @@ +use std::collections::HashSet; + +use regex::Regex; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use snafu::{ResultExt as _, Snafu}; + +#[cfg(doc)] +use crate::config::merge::Merge; + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum Error { + #[snafu(display("missing roleGroup {role_group:?}"))] + MissingRoleGroup { role_group: String }, + + #[snafu(display( + "Could not parse regex from \"jvmArgumentOverrides.removeRegex\", ignoring it (there might be some added anchors at the start and end): {regex:?}" + ))] + InvalidRemoveRegex { source: regex::Error, regex: String }, +} + +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JavaCommonConfig { + /// Allows overriding JVM arguments. + // + /// Please read on the [JVM argument overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#jvm-argument-overrides) + /// for details on the usage. + #[serde(default)] + pub jvm_argument_overrides: JvmArgumentOverrides, +} + +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JvmArgumentOverrides { + /// JVM arguments to be added + #[serde(default)] + add: Vec, + + /// JVM arguments to be removed by exact match + // + // HashSet to be optimized for quick lookup + #[serde(default)] + remove: HashSet, + + /// JVM arguments matching any of this regexes will be removed + #[serde(default)] + remove_regex: Vec, +} + +impl JvmArgumentOverrides { + pub fn new(add: Vec, remove: HashSet, remove_regex: Vec) -> Self { + Self { + add, + remove, + remove_regex, + } + } + + pub fn new_with_only_additions(add: Vec) -> Self { + Self { + add, + ..Default::default() + } + } + + /// Called on **merged** [`JvmArgumentOverrides`}, returns all arguments that should be passed to the JVM. + /// + /// **Can only be called on merged config, it will panic otherwise!** + /// + /// We are panicking (instead of returning an Error), because this is not the users fault, but + /// the operator is doing things wrong + pub fn effective_jvm_config_after_merging(&self) -> &Vec { + assert!( + self.remove.is_empty(), + "After merging there should be no removals left. \"effective_jvm_config_after_merging\" should only be called on merged configs!" + ); + assert!( + self.remove_regex.is_empty(), + "After merging there should be no removal regexes left. \"effective_jvm_config_after_merging\" should only be called on merged configs!" + ); + + &self.add + } +} + +/// We can not use [`Merge`] here, as this function can fail, e.g. if an invalid regex is specified by the user +impl JvmArgumentOverrides { + /// Please watch out: Merge order is complicated for this merge. + /// Test your code! + pub fn try_merge(&mut self, defaults: &Self) -> Result<(), Error> { + let regexes = self + .remove_regex + .iter() + .map(|regex| { + let without_anchors = regex.trim_start_matches('^').trim_end_matches('$'); + let with_anchors = format!("^{without_anchors}$"); + + Regex::new(&with_anchors).with_context(|_| InvalidRemoveRegexSnafu { + regex: with_anchors, + }) + }) + .collect::, Error>>()?; + + let new_add = defaults + .add + .iter() + .filter(|arg| !self.remove.contains(*arg)) + .filter(|arg| !regexes.iter().any(|regex| regex.is_match(arg))) + .chain(self.add.iter()) + .cloned() + .collect(); + + self.add = new_add; + self.remove = HashSet::new(); + self.remove_regex = Vec::new(); + + Ok(()) + } +} + +#[cfg(feature = "crds")] +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + use crate::{ + crd::role::{GenericRoleConfig, v1alpha2::Role}, + test_utils::deserialize_from_yaml_with_singleton_map, + }; + + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + struct EmptyConfigOverrides {} + + #[test] + fn test_merge_java_common_config() { + // The operator generates some JVM arguments + let operator_generated = JvmArgumentOverrides::new_with_only_additions( + [ + "-Xms34406m".to_owned(), + "-Xmx34406m".to_owned(), + "-XX:+UseG1GC".to_owned(), + "-XX:+ExitOnOutOfMemoryError".to_owned(), + "-Djava.protocol.handler.pkgs=sun.net.www.protocol".to_owned(), + "-Dsun.net.http.allowRestrictedHeaders=true".to_owned(), + "-Djava.security.properties=/stackable/nifi/conf/security.properties".to_owned(), + ] + .into(), + ); + + let entire_role: Role<(), EmptyConfigOverrides, GenericRoleConfig, JavaCommonConfig> = + deserialize_from_yaml_with_singleton_map(" + # Let's say we want to set some additional HTTP Proxy and IPv4 settings + # And we don't like the garbage collector for some reason... + jvmArgumentOverrides: + remove: + - -XX:+UseG1GC + add: # Add some networking arguments + - -Dhttps.proxyHost=proxy.my.corp + - -Dhttps.proxyPort=8080 + - -Djava.net.preferIPv4Stack=true + roleGroups: + default: + # Replicas is required since v1alpha2 RoleGroup. + replicas: + fixed: + count: 1 + # For the roleGroup, let's say we need a different memory config. + # For that to work we first remove the flags generated by the operator and add our own. + # Also we override the proxy port to test that the roleGroup config takes precedence over the role config. + jvmArgumentOverrides: + removeRegex: + - -Xmx.* + - -Dhttps.proxyPort=.* + add: + - -Xmx40000m + - -Dhttps.proxyPort=1234 + ") + .expect("Failed to parse role"); + + let merged_jvm_argument_overrides = entire_role + .get_merged_jvm_argument_overrides("default", &operator_generated) + .expect("Failed to merge jvm argument overrides"); + + let expected = Vec::from([ + "-Xms34406m".to_owned(), + "-XX:+ExitOnOutOfMemoryError".to_owned(), + "-Djava.protocol.handler.pkgs=sun.net.www.protocol".to_owned(), + "-Dsun.net.http.allowRestrictedHeaders=true".to_owned(), + "-Djava.security.properties=/stackable/nifi/conf/security.properties".to_owned(), + "-Dhttps.proxyHost=proxy.my.corp".to_owned(), + "-Djava.net.preferIPv4Stack=true".to_owned(), + "-Xmx40000m".to_owned(), + "-Dhttps.proxyPort=1234".to_owned(), + ]); + + assert_eq!( + merged_jvm_argument_overrides, + JvmArgumentOverrides { + add: expected.clone(), + remove: HashSet::new(), + remove_regex: Vec::new() + } + ); + + assert_eq!( + merged_jvm_argument_overrides.effective_jvm_config_after_merging(), + &expected + ); + } + + #[test] + fn test_merge_java_common_config_keep_order() { + let operator_generated = + JvmArgumentOverrides::new_with_only_additions(["-Xms1m".to_owned()].into()); + + let entire_role: Role<(), EmptyConfigOverrides, GenericRoleConfig, JavaCommonConfig> = + deserialize_from_yaml_with_singleton_map( + " + jvmArgumentOverrides: + add: + - -Xms2m + roleGroups: + default: + # Replicas is required since v1alpha2 RoleGroup. + replicas: + fixed: + count: 1 + jvmArgumentOverrides: + add: + - -Xms3m + ", + ) + .expect("role must parse"); + + let merged_jvm_argument_overrides = entire_role + .get_merged_jvm_argument_overrides("default", &operator_generated) + .expect("Failed to merge jvm argument overrides"); + + assert_eq!( + merged_jvm_argument_overrides.effective_jvm_config_after_merging(), + &[ + "-Xms1m".to_owned(), + "-Xms2m".to_owned(), + "-Xms3m".to_owned() + ] + ); + } +} diff --git a/crates/stackable-operator/src/crd/role/mod.rs b/crates/stackable-operator/src/crd/role/mod.rs new file mode 100644 index 000000000..362baa8d3 --- /dev/null +++ b/crates/stackable-operator/src/crd/role/mod.rs @@ -0,0 +1,399 @@ +//! This module provides utility functions for dealing with role (types) and role groups. +//! +//! While other modules in this crate try to be generic and reusable for other operators this one makes very specific +//! assumptions about how a CRD is structured. +//! +//! These assumptions are detailed and explained below. +//! +//! ## Roles / Role types +//! +//! A CRD is often used to operate another piece of software. +//! Software - especially the distributed kind - sometimes consists of multiple different types of program working +//! together to achieve their goal. These different types are what we call a _role_. +//! +//! ## Examples +//! +//! ### Apache Hadoop HDFS +//! +//! - NameNode +//! - DataNode +//! - JournalNode +//! +//! ### Kubernetes +//! +//! - kube-apiserver +//! - kubelet +//! - kube-controller-manager +//! - ... +//! +//! ## Role Groups +//! +//! There is sometimes a need to have different configuration options or different label selectors for different replicas of the same role. +//! Role groups are what allows this. +//! Nested under a role there can be multiple role groups, each with its own LabelSelector and configuration. +//! +//! ### Example +//! +//! This example has one role (`leader`) and two role groups (`default`, and `20core`) +//! +//! ```yaml +//! leader: +//! roleGroups: +//! default: +//! selector: +//! matchLabels: +//! component: spark +//! matchExpressions: +//! - { key: tier, operator: In, values: [ cache ] } +//! - { key: environment, operator: NotIn, values: [ dev ] } +//! config: +//! cores: 1 +//! memory: "1g" +//! replicas: 3 +//! 20core: +//! selector: +//! matchLabels: +//! component: spark +//! cores: 20 +//! matchExpressions: +//! - { key: tier, operator: In, values: [ cache ] } +//! - { key: environment, operator: NotIn, values: [ dev ] } +//! config: +//! cores: 10 +//! memory: "1g" +//! replicas: 3 +//! config: +//! ``` +//! +//! ## Pod labels +//! +//! Each Pod that Operators create needs to have a common set of labels. +//! These labels are (with one exception) listed in the Kubernetes [documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/): +//! +//! - `app.kubernetes.io/name`: The name of the application. This will usually be a static string (e.g. "zookeeper"). +//! - `app.kubernetes.io/instance`: The name of the parent resource, this is useful so an operator can list all its pods by using a LabelSelector +//! - `app.kubernetes.io/version`: The current version of the application +//! - `app.kubernetes.io/component`: The role/role type, this is used to distinguish multiple pods on the same node from each other +//! - `app.kubernetes.io/part-of`: The name of a higher level application this one is part of. We have decided to leave this empty for now. +//! - `app.kubernetes.io/managed-by`: The tool being used to manage the operation of an application (e.g. "zookeeper-operator") +//! - `app.kubernetes.io/role-group`: The name of the role group this pod belongs to +//! +//! NOTE: We find the official description to be ambiguous so we use these labels as defined above. +//! +//! Each resource can have more operator specific labels. + +use std::collections::{BTreeMap, HashMap}; + +use k8s_openapi::api::core::v1::PodTemplateSpec; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "crds")] +use crate::versioned::versioned; +use crate::{commons::pdb::PdbConfig, utils::crds::raw_object_schema}; + +mod java; +pub use java::{Error, JavaCommonConfig, JvmArgumentOverrides}; + +mod utils; +pub use utils::*; + +#[cfg(feature = "crds")] +mod v1alpha2_impl; + +#[cfg(feature = "crds")] +#[versioned(version(name = "v1alpha1"), version(name = "v1alpha2"))] +pub mod versioned { + /// This struct represents a role - e.g. HDFS datanodes or Trino workers. + /// + /// It has a key-value-map containing all the roleGroups that are part of this role. Additionally, there is a + /// `config`, which is configurable at the role **and** roleGroup level. Everything at roleGroup level is merged on + /// top of what is configured on role level. + /// + /// There is also a second form of config, which can only be configured at role level, the `roleConfig`. You can + /// learn more about this in the [Roles and role group concept documentation][1]. + /// + /// [1]: DOCS_BASE_URL_PLACEHOLDER/concepts/roles-and-role-groups + // + // Everything below is only a "normal" comment, not rustdoc - so we don't bloat the CRD documentation + // with technical (Rust) details. + // + // `Config` here is the `config` shared between role and roleGroup. + // + // `RoleConfig` here is the `roleConfig` only available on the role. It defaults to [`GenericRoleConfig`], which is + // sufficient for most of the products. There are some exceptions, where e.g. [`EmptyRoleConfig`] is used. + // However, product-operators can define their own - custom - struct and use that here. + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct Role< + Config, + ConfigOverrides, + RoleConfig = GenericRoleConfig, + CommonConfig = GenericCommonConfig, + > + where + // Don't remove this trait bounds!!! + // We don't know why, but if you remove either of them, the generated default value in the CRDs will + // be missing! + RoleConfig: Default + JsonSchema + Serialize, + CommonConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, + { + #[serde( + flatten, + bound( + deserialize = "Config: Default + Deserialize<'de>, CommonConfig: Deserialize<'de>, ConfigOverrides: Deserialize<'de>" + ) + )] + pub config: CommonConfiguration, + + #[serde(default)] + pub role_config: RoleConfig, + + /// The set of role groups for this role, keyed by their name. + /// + /// A role group is a subset of the replicas of a role that share the same configuration, + /// allowing finer-grained control than the role level. This is useful to e.g. schedule groups + /// onto different classes of nodes or into different regions, or to run them with different + /// settings. Configuration set on a role group is merged on top of the role-level `config`, + /// with the more specific role group values taking precedence. + /// + /// Every role needs at least one role group. A role with a single role group conventionally + /// names it `default`. + /// + /// Read the + /// [roles and role groups concept documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/roles-and-role-groups) + /// for more details. + #[versioned( + changed( + since = "v1alpha2", + from_type = "HashMap>" + ), + hint(map) + )] + pub role_groups: + HashMap>, + } + + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde( + rename_all = "camelCase", + bound( + deserialize = "Config: Default + Deserialize<'de>, CommonConfig: Default + Deserialize<'de>, ConfigOverrides: Default + Deserialize<'de>" + ) + )] + #[schemars( + bound = "Config: JsonSchema, CommonConfig: JsonSchema, ConfigOverrides: Default + JsonSchema" + )] + pub struct RoleGroup { + #[serde(flatten)] + pub config: CommonConfiguration, + + #[versioned(changed(since = "v1alpha2", from_type = "Option"))] + pub replicas: Replicas, + } +} + +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde( + rename_all = "camelCase", + bound( + deserialize = "Config: Default + Deserialize<'de>, CommonConfig: Default + Deserialize<'de>, ConfigOverrides: Default + Deserialize<'de>" + ) +)] +#[schemars( + bound = "Config: JsonSchema, CommonConfig: JsonSchema, ConfigOverrides: Default + JsonSchema" +)] +pub struct CommonConfiguration { + #[serde(default)] + // We can't depend on Config being `Default`, since that trait is not object-safe + // We only need to generate schemas for fully specified types, but schemars_derive + // does not support specifying custom bounds. + #[schemars(default = "Self::default_config")] + pub config: Config, + + /// The `configOverrides` can be used to configure properties in product config files + /// that are not exposed in the CRD. Read the + /// [config overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#config-overrides) + /// and consult the operator specific usage guide documentation for details on the + /// available config files and settings for the specific product. + #[serde(default)] + pub config_overrides: ConfigOverrides, + + /// `envOverrides` configure environment variables to be set in the Pods. + /// It is a map from strings to strings - environment variables and the value to set. + /// Read the + /// [environment variable overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#env-overrides) + /// for more information and consult the operator specific usage guide to find out about + /// the product specific environment variables that are available. + #[serde(default)] + pub env_overrides: HashMap, + + // BTreeMap to keep some order with the cli arguments. + // TODO add documentation. + #[serde(default)] + pub cli_overrides: BTreeMap, + + /// In the `podOverrides` property you can define a + /// [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + /// to override any property that can be set on a Kubernetes Pod. + /// Read the + /// [Pod overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#pod-overrides) + /// for more information. + #[serde(default)] + #[schemars(schema_with = "raw_object_schema")] + pub pod_overrides: PodTemplateSpec, + + // No docs needed, as we flatten this struct. + // + // This field is product-specific and can contain e.g. jvmArgumentOverrides. + // + // If [`JavaCommonConfig`] is used, please use [`Role::get_merged_jvm_argument_overrides`] instead of + // reading this field directly. + #[serde(flatten, default)] + pub product_specific_common_config: CommonConfig, +} + +impl + CommonConfiguration +{ + fn default_config() -> serde_json::Value { + serde_json::json!({}) + } +} + +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +pub struct GenericCommonConfig {} + +/// This is a product-agnostic RoleConfig, which is sufficient for most of the products. +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GenericRoleConfig { + #[serde(default)] + pub pod_disruption_budget: PdbConfig, +} + +/// This is a product-agnostic RoleConfig, with nothing in it. It is used e.g. by products that have +/// nothing configurable at role level. +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct EmptyRoleConfig {} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Replicas { + /// Use a fixed number of replicas. + // + // We sadly cannot simply wrap a u16 because Kubernetes doesn't support mixing enum variants + // with the shape of an object AND primitive types, like a u16 in this case. + Fixed { count: u16 }, + + /// Use a fully managed, auto-scaled number of replicas. + Managed {}, + + /// Use a auto-scaled number of replicas managed by external/custom mechanisms, e.g. HPA. + Custom {}, +} + +impl From> for Replicas { + fn from(replicas: Option) -> Self { + match replicas { + Some(count) => Self::Fixed { count }, + None => Self::Custom {}, + } + } +} + +impl From for Option { + fn from(replicas: Replicas) -> Self { + match replicas { + Replicas::Fixed { count } => Some(count), + Replicas::Managed {} => None, + // FIXME (@Techassi): Deal with this (we need fallible conversions or we need to support + // tracking type changes). Alternatively, we could map this to None as well, but that + // would be a lossy conversion. + Replicas::Custom {} => todo!(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replica_counts_with_all_replicas_set() { + let replicas = [Some(3), Some(2), Some(5)]; + + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), + Some(10) + ); + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), + Some(10) + ); + assert_eq!(estimated_replica_count(replicas), 10); + } + + #[test] + fn replica_counts_with_one_replica_unset() { + let replicas = [Some(3), None, Some(2)]; + + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), + None + ); + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), + None + ); + assert_eq!(estimated_replica_count(replicas), 6); + } + + #[test] + fn replica_counts_with_a_zero_replica() { + let replicas = [Some(3), Some(0)]; + + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), + Some(3) + ); + // With treat_zero_as_none the zero turns the whole count into None. + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), + None + ); + assert_eq!(estimated_replica_count(replicas), 3); + } + + #[test] + fn replica_counts_with_a_single_zero_role_groups_group() { + let replicas = [Some(0)]; + + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), + Some(0) + ); + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), + None + ); + assert_eq!(estimated_replica_count(replicas), 0); + } + + #[test] + fn replica_counts_without_role_groups() { + let replicas = []; + + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), + Some(0) + ); + assert_eq!( + fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), + None + ); + assert_eq!(estimated_replica_count(replicas), 0); + } +} diff --git a/crates/stackable-operator/src/crd/role/utils.rs b/crates/stackable-operator/src/crd/role/utils.rs new file mode 100644 index 000000000..623913da7 --- /dev/null +++ b/crates/stackable-operator/src/crd/role/utils.rs @@ -0,0 +1,105 @@ +use std::fmt::Display; + +use educe::Educe; +use kube::{Resource, runtime::reflector::ObjectRef}; + +/// A reference to a named role group of a given cluster object +#[derive(Educe)] +#[educe(Clone, Debug)] +pub struct RoleGroupRef { + pub cluster: ObjectRef, + pub role: String, + pub role_group: String, +} + +impl RoleGroupRef { + pub fn object_name(&self) -> String { + format!("{}-{}-{}", self.cluster.name, self.role, self.role_group) + } + + /// Returns the service name used by rolegroups for cluster internal communication only. + /// + /// The internal use of of this service name is indicated by the `-headless` suffix. + /// This service should not be used for communication to external services or clients + /// and also should not be used to export metrics (like Prometheus). Metrics should be + /// instead exposed via a dedicated service. Use [`Self::rolegroup_metrics_service_name`] + /// instead. + pub fn rolegroup_headless_service_name(&self) -> String { + format!("{name}-headless", name = self.object_name()) + } + + /// Returns the service name used by rolegroups to expose metrics (like Prometheus). + /// + /// The use for metrics only is indicated by the `-metrics` suffix. This service + /// should not be used for any internal communication or any other external + /// communication. For internal communication, use [`Self::rolegroup_headless_service_name`] + /// instead. + pub fn rolegroup_metrics_service_name(&self) -> String { + format!("{name}-metrics", name = self.object_name()) + } +} + +impl Display for RoleGroupRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "role group {}/{} of {}", + self.role, self.role_group, self.cluster + )) + } +} + +/// Returns [`Some`] in case the number of replicas is hard-coded to a certain value. +/// +/// This is the case when all `replicas` are set to [`Some`], in which case they are simply +/// summed. +/// +/// The argument `zero_replicas_counting` is a safety mechanism, which allows the caller to decide +/// if an explicit replica count of `0` should be treated as [`None`]. It also means that [`None`] +/// is returned in case no roleGroups are configured at all. +// +// Note: We are using a [`IntoIterator`] combined with `.peekable()` over [`ExactSizeIterator`] to +// have minimal bound requirements on the caller. +pub fn fixed_replica_count>>( + replicas: I, + zero_replicas_counting: ZeroReplicasCounting, +) -> Option { + let mut replicas = replicas.into_iter().peekable(); + + // An empty role has no fixed replica count when zeros are treated as None. + if zero_replicas_counting == ZeroReplicasCounting::TreatAsNone && replicas.peek().is_none() { + return None; + } + + replicas + .map(|replicas| match replicas { + None => None, + Some(0) if zero_replicas_counting == ZeroReplicasCounting::TreatAsNone => None, + // The individual replicas are [`u16`]s, so a [`u32`] sum has plenty of space. + Some(replicas) => Some(u32::from(replicas)), + }) + .sum() +} + +/// Returns the estimated total number of replicas across all role groups. +/// +/// Unlike [`fixed_replica_count`], this always returns a value: a role group with an unset (i.e. +/// [`None`]) replica count is assumed to run a single replica. Use this when a best-effort estimate +/// is needed even though the exact number of replicas is not hard-coded. +// +// Note: We are using a [`IntoIterator`] combined with `.peekable()` over [`ExactSizeIterator`] to +// have minimal bound requirements on the caller. +pub fn estimated_replica_count>>(replicas: I) -> u32 { + replicas + .into_iter() + .map(|replicas| u32::from(replicas.unwrap_or(1))) + .sum() +} + +/// How explicit zero (`0`) replicas on a role group should be counted +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ZeroReplicasCounting { + /// Treat them as what they are: `Some(0)`. + TreatAsZero, + /// Treat them as if the user configured [`None`]. + TreatAsNone, +} diff --git a/crates/stackable-operator/src/crd/role/v1alpha2_impl.rs b/crates/stackable-operator/src/crd/role/v1alpha2_impl.rs new file mode 100644 index 000000000..0361827ca --- /dev/null +++ b/crates/stackable-operator/src/crd/role/v1alpha2_impl.rs @@ -0,0 +1,74 @@ +use schemars::JsonSchema; +use serde::Serialize; +use snafu::OptionExt as _; + +use crate::{ + config::{ + fragment::{self, FromFragment}, + merge::Merge, + }, + crd::role::{ + java::{Error, JavaCommonConfig, JvmArgumentOverrides, MissingRoleGroupSnafu}, + v1alpha2::{Role, RoleGroup}, + }, +}; + +impl + Role +where + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, +{ + /// Merges jvm argument overrides from + /// + /// 1. It takes the operator generated JVM args + /// 2. It applies role level overrides + /// 3. It applies roleGroup level overrides + pub fn get_merged_jvm_argument_overrides( + &self, + role_group: &str, + operator_generated: &JvmArgumentOverrides, + ) -> Result { + let from_role = &self + .config + .product_specific_common_config + .jvm_argument_overrides; + let from_role_group = &self + .role_groups + .get(role_group) + .with_context(|| MissingRoleGroupSnafu { role_group })? + .config + .product_specific_common_config + .jvm_argument_overrides; + + // Please note that the merge order is different than we normally do! + // This is not trivial, as the merge operation is not purely additive (as it is with e.g. `PodTemplateSpec). + let mut from_role = from_role.clone(); + from_role.try_merge(operator_generated)?; + let mut from_role_group = from_role_group.clone(); + from_role_group.try_merge(&from_role)?; + + Ok(from_role_group) + } +} + +impl RoleGroup { + pub fn validate_config( + &self, + role: &Role, + default_config: &Config, + ) -> Result + where + C: FromFragment, + Config: Merge + Clone, + RoleConfig: Default + JsonSchema + Serialize, + CommonConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, + { + let mut role_config = role.config.config.clone(); + role_config.merge(default_config); + let mut rolegroup_config = self.config.config.clone(); + rolegroup_config.merge(&role_config); + fragment::validate(rolegroup_config) + } +} diff --git a/crates/stackable-operator/src/lib.rs b/crates/stackable-operator/src/lib.rs index e52cd0382..d74259784 100644 --- a/crates/stackable-operator/src/lib.rs +++ b/crates/stackable-operator/src/lib.rs @@ -15,7 +15,6 @@ pub mod config; pub mod config_overrides; pub mod constants; pub mod cpu; -#[cfg(feature = "crds")] pub mod crd; pub mod database_connections; pub mod deep_merger; @@ -28,7 +27,6 @@ pub mod memory; pub mod namespace; pub mod pod_utils; pub mod product_logging; -pub mod role_utils; pub mod status; pub mod test_utils; pub mod utils; @@ -51,3 +49,25 @@ pub use stackable_telemetry as telemetry; pub use stackable_versioned as versioned; #[cfg(feature = "webhook")] pub use stackable_webhook as webhook; + +/// This macro can be used to gate every contained item behind `feature`. +/// +/// ```ignore +/// use crate::cfg_block; +/// +/// cfg_block! { +/// "crds"; +/// +/// pub use foo; +/// pub use bar; +/// } +/// ``` +#[macro_export] +macro_rules! cfg_block { + ($feature:literal; $($item:item)*) => { + $( + #[cfg(feature = $feature)] + $item + )* + } +} diff --git a/crates/stackable-operator/src/product_logging/framework.rs b/crates/stackable-operator/src/product_logging/framework.rs index 35c7bab02..22223e716 100644 --- a/crates/stackable-operator/src/product_logging/framework.rs +++ b/crates/stackable-operator/src/product_logging/framework.rs @@ -7,6 +7,7 @@ use snafu::{ResultExt, Snafu}; use crate::{ builder::{self, pod::container::ContainerBuilder}, commons::product_image_selection::ResolvedProductImage, + crd::role::RoleGroupRef, k8s_openapi::{ api::core::v1::{Container, ResourceRequirements}, apimachinery::pkg::api::resource::Quantity, @@ -16,7 +17,6 @@ use crate::{ product_logging::spec::{ AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, LogLevel, }, - role_utils::RoleGroupRef, }; /// Config directory used in the Vector log agent container @@ -653,7 +653,7 @@ pub fn create_logback_config( /// # k8s_openapi::api::core::v1::Pod, /// # kube::runtime::reflector::ObjectRef, /// # product_logging::spec::default_logging, -/// # role_utils::RoleGroupRef, +/// # crd::role::RoleGroupRef, /// # }; /// # use strum::{Display, EnumIter}; /// # diff --git a/crates/stackable-operator/src/role_utils.rs b/crates/stackable-operator/src/role_utils.rs deleted file mode 100644 index 13b692e06..000000000 --- a/crates/stackable-operator/src/role_utils.rs +++ /dev/null @@ -1,729 +0,0 @@ -//! This module provides utility functions for dealing with role (types) and role groups. -//! -//! While other modules in this crate try to be generic and reusable for other operators -//! this one makes very specific assumptions about how a CRD is structured. -//! -//! These assumptions are detailed and explained below. -//! -//! # Roles / Role types -//! -//! A CRD is often used to operate another piece of software. -//! Software - especially the distributed kind - sometimes consists of multiple different types of program working together to achieve their goal. -//! These different types are what we call a _role_. -//! -//! ## Examples -//! -//! Apache Hadoop HDFS: -//! * NameNode -//! * DataNode -//! * JournalNode -//! -//! Kubernetes: -//! * kube-apiserver -//! * kubelet -//! * kube-controller-manager -//! * ... -//! -//! # Role Groups -//! -//! There is sometimes a need to have different configuration options or different label selectors for different replicas of the same role. -//! Role groups are what allows this. -//! Nested under a role there can be multiple role groups, each with its own LabelSelector and configuration. -//! -//! ## Example -//! -//! This example has one role (`leader`) and two role groups (`default`, and `20core`) -//! -//! ```yaml -//! leader: -//! roleGroups: -//! default: -//! selector: -//! matchLabels: -//! component: spark -//! matchExpressions: -//! - { key: tier, operator: In, values: [ cache ] } -//! - { key: environment, operator: NotIn, values: [ dev ] } -//! config: -//! cores: 1 -//! memory: "1g" -//! replicas: 3 -//! 20core: -//! selector: -//! matchLabels: -//! component: spark -//! cores: 20 -//! matchExpressions: -//! - { key: tier, operator: In, values: [ cache ] } -//! - { key: environment, operator: NotIn, values: [ dev ] } -//! config: -//! cores: 10 -//! memory: "1g" -//! replicas: 3 -//! config: -//! ``` -//! -//! # Pod labels -//! -//! Each Pod that Operators create needs to have a common set of labels. -//! These labels are (with one exception) listed in the Kubernetes [documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/): -//! -//! * app.kubernetes.io/name - The name of the application. This will usually be a static string (e.g. "zookeeper"). -//! * app.kubernetes.io/instance - The name of the parent resource, this is useful so an operator can list all its pods by using a LabelSelector -//! * app.kubernetes.io/version - The current version of the application -//! * app.kubernetes.io/component - The role/role type, this is used to distinguish multiple pods on the same node from each other -//! * app.kubernetes.io/part-of - The name of a higher level application this one is part of. We have decided to leave this empty for now. -//! * app.kubernetes.io/managed-by - The tool being used to manage the operation of an application (e.g. "zookeeper-operator") -//! * app.kubernetes.io/role-group - The name of the role group this pod belongs to -//! -//! NOTE: We find the official description to be ambiguous so we use these labels as defined above. -//! -//! Each resource can have more operator specific labels. - -use std::{ - collections::{BTreeMap, HashMap, HashSet}, - fmt::{Debug, Display}, -}; - -use educe::Educe; -use k8s_openapi::api::core::v1::PodTemplateSpec; -use kube::{Resource, runtime::reflector::ObjectRef}; -use regex::Regex; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use snafu::{OptionExt, ResultExt, Snafu}; - -use crate::{ - commons::pdb::PdbConfig, - config::{ - fragment::{self, FromFragment}, - merge::Merge, - }, - utils::crds::raw_object_schema, -}; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("missing roleGroup {role_group:?}"))] - MissingRoleGroup { role_group: String }, - - #[snafu(display( - "Could not parse regex from \"jvmArgumentOverrides.removeRegex\", ignoring it (there might be some added anchors at the start and end): {regex:?}" - ))] - InvalidRemoveRegex { source: regex::Error, regex: String }, -} - -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde( - rename_all = "camelCase", - bound( - deserialize = "Config: Default + Deserialize<'de>, CommonConfig: Default + Deserialize<'de>, ConfigOverrides: Default + Deserialize<'de>" - ) -)] -#[schemars( - bound = "Config: JsonSchema, CommonConfig: JsonSchema, ConfigOverrides: Default + JsonSchema" -)] -pub struct CommonConfiguration { - #[serde(default)] - // We can't depend on Config being `Default`, since that trait is not object-safe - // We only need to generate schemas for fully specified types, but schemars_derive - // does not support specifying custom bounds. - #[schemars(default = "Self::default_config")] - pub config: Config, - - /// The `configOverrides` can be used to configure properties in product config files - /// that are not exposed in the CRD. Read the - /// [config overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#config-overrides) - /// and consult the operator specific usage guide documentation for details on the - /// available config files and settings for the specific product. - #[serde(default)] - pub config_overrides: ConfigOverrides, - - /// `envOverrides` configure environment variables to be set in the Pods. - /// It is a map from strings to strings - environment variables and the value to set. - /// Read the - /// [environment variable overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#env-overrides) - /// for more information and consult the operator specific usage guide to find out about - /// the product specific environment variables that are available. - #[serde(default)] - pub env_overrides: HashMap, - - // BTreeMap to keep some order with the cli arguments. - // TODO add documentation. - #[serde(default)] - pub cli_overrides: BTreeMap, - - /// In the `podOverrides` property you can define a - /// [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) - /// to override any property that can be set on a Kubernetes Pod. - /// Read the - /// [Pod overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#pod-overrides) - /// for more information. - #[serde(default)] - #[schemars(schema_with = "raw_object_schema")] - pub pod_overrides: PodTemplateSpec, - - // No docs needed, as we flatten this struct. - // - // This field is product-specific and can contain e.g. jvmArgumentOverrides. - // - // If [`JavaCommonConfig`] is used, please use [`Role::get_merged_jvm_argument_overrides`] instead of - // reading this field directly. - #[serde(flatten, default)] - pub product_specific_common_config: CommonConfig, -} - -impl - CommonConfiguration -{ - fn default_config() -> serde_json::Value { - serde_json::json!({}) - } -} - -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] -pub struct GenericCommonConfig {} - -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct JavaCommonConfig { - /// Allows overriding JVM arguments. - // - /// Please read on the [JVM argument overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#jvm-argument-overrides) - /// for details on the usage. - #[serde(default)] - pub jvm_argument_overrides: JvmArgumentOverrides, -} - -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct JvmArgumentOverrides { - /// JVM arguments to be added - #[serde(default)] - add: Vec, - - /// JVM arguments to be removed by exact match - // - // HashSet to be optimized for quick lookup - #[serde(default)] - remove: HashSet, - - /// JVM arguments matching any of this regexes will be removed - #[serde(default)] - remove_regex: Vec, -} - -impl JvmArgumentOverrides { - pub fn new(add: Vec, remove: HashSet, remove_regex: Vec) -> Self { - Self { - add, - remove, - remove_regex, - } - } - - pub fn new_with_only_additions(add: Vec) -> Self { - Self { - add, - ..Default::default() - } - } - - /// Called on **merged** [`JvmArgumentOverrides`}, returns all arguments that should be passed to the JVM. - /// - /// **Can only be called on merged config, it will panic otherwise!** - /// - /// We are panicking (instead of returning an Error), because this is not the users fault, but - /// the operator is doing things wrong - pub fn effective_jvm_config_after_merging(&self) -> &Vec { - assert!( - self.remove.is_empty(), - "After merging there should be no removals left. \"effective_jvm_config_after_merging\" should only be called on merged configs!" - ); - assert!( - self.remove_regex.is_empty(), - "After merging there should be no removal regexes left. \"effective_jvm_config_after_merging\" should only be called on merged configs!" - ); - - &self.add - } -} - -/// We can not use [`Merge`] here, as this function can fail, e.g. if an invalid regex is specified by the user -impl JvmArgumentOverrides { - /// Please watch out: Merge order is complicated for this merge. - /// Test your code! - pub fn try_merge(&mut self, defaults: &Self) -> Result<(), Error> { - let regexes = self - .remove_regex - .iter() - .map(|regex| { - let without_anchors = regex.trim_start_matches('^').trim_end_matches('$'); - let with_anchors = format!("^{without_anchors}$"); - - Regex::new(&with_anchors).with_context(|_| InvalidRemoveRegexSnafu { - regex: with_anchors, - }) - }) - .collect::, Error>>()?; - - let new_add = defaults - .add - .iter() - .filter(|arg| !self.remove.contains(*arg)) - .filter(|arg| !regexes.iter().any(|regex| regex.is_match(arg))) - .chain(self.add.iter()) - .cloned() - .collect(); - - self.add = new_add; - self.remove = HashSet::new(); - self.remove_regex = Vec::new(); - - Ok(()) - } -} - -/// This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a key-value-map containing -/// all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable -/// at the role *and* roleGroup level. Everything at roleGroup level is merged on top of what is configured -/// on role level. There is also a second form of config, which can only be configured -/// at role level, the `roleConfig`. -/// You can learn more about this in the -/// [Roles and role group concept documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/roles-and-role-groups). -// -// Everything below is only a "normal" comment, not rustdoc - so we don't bloat the CRD documentation -// with technical (Rust) details. -// -// `Config` here is the `config` shared between role and roleGroup. -// -// `RoleConfig` here is the `roleConfig` only available on the role. It defaults to [`GenericRoleConfig`], which is -// sufficient for most of the products. There are some exceptions, where e.g. [`EmptyRoleConfig`] is used. -// However, product-operators can define their own - custom - struct and use that here. -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct Role< - Config, - ConfigOverrides, - RoleConfig = GenericRoleConfig, - CommonConfig = GenericCommonConfig, -> where - // Don't remove this trait bounds!!! - // We don't know why, but if you remove either of them, the generated default value in the CRDs will - // be missing! - RoleConfig: Default + JsonSchema + Serialize, - CommonConfig: Default + JsonSchema + Serialize, - ConfigOverrides: Default + JsonSchema + Serialize, -{ - #[serde( - flatten, - bound( - deserialize = "Config: Default + Deserialize<'de>, CommonConfig: Deserialize<'de>, ConfigOverrides: Deserialize<'de>" - ) - )] - pub config: CommonConfiguration, - - #[serde(default)] - pub role_config: RoleConfig, - - /// The set of role groups for this role, keyed by their name. - /// - /// A role group is a subset of the replicas of a role that share the same configuration, - /// allowing finer-grained control than the role level. This is useful to e.g. schedule groups - /// onto different classes of nodes or into different regions, or to run them with different - /// settings. Configuration set on a role group is merged on top of the role-level `config`, - /// with the more specific role group values taking precedence. - /// - /// Every role needs at least one role group. A role with a single role group conventionally - /// names it `default`. - /// - /// Read the - /// [roles and role groups concept documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/roles-and-role-groups) - /// for more details. - pub role_groups: HashMap>, -} - -impl - Role -where - RoleConfig: Default + JsonSchema + Serialize, - ConfigOverrides: Default + JsonSchema + Serialize, -{ - /// Merges jvm argument overrides from - /// - /// 1. It takes the operator generated JVM args - /// 2. It applies role level overrides - /// 3. It applies roleGroup level overrides - pub fn get_merged_jvm_argument_overrides( - &self, - role_group: &str, - operator_generated: &JvmArgumentOverrides, - ) -> Result { - let from_role = &self - .config - .product_specific_common_config - .jvm_argument_overrides; - let from_role_group = &self - .role_groups - .get(role_group) - .with_context(|| MissingRoleGroupSnafu { role_group })? - .config - .product_specific_common_config - .jvm_argument_overrides; - - // Please note that the merge order is different than we normally do! - // This is not trivial, as the merge operation is not purely additive (as it is with e.g. `PodTemplateSpec). - let mut from_role = from_role.clone(); - from_role.try_merge(operator_generated)?; - let mut from_role_group = from_role_group.clone(); - from_role_group.try_merge(&from_role)?; - - Ok(from_role_group) - } -} - -/// This is a product-agnostic RoleConfig, which is sufficient for most of the products. -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct GenericRoleConfig { - #[serde(default)] - pub pod_disruption_budget: PdbConfig, -} - -/// This is a product-agnostic RoleConfig, with nothing in it. It is used e.g. by products that have -/// nothing configurable at role level. -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct EmptyRoleConfig {} - -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde( - rename_all = "camelCase", - bound( - deserialize = "Config: Default + Deserialize<'de>, CommonConfig: Default + Deserialize<'de>, ConfigOverrides: Default + Deserialize<'de>" - ) -)] -#[schemars( - bound = "Config: JsonSchema, CommonConfig: JsonSchema, ConfigOverrides: Default + JsonSchema" -)] -pub struct RoleGroup { - #[serde(flatten)] - pub config: CommonConfiguration, - pub replicas: Option, -} - -impl RoleGroup { - pub fn validate_config( - &self, - role: &Role, - default_config: &Config, - ) -> Result - where - C: FromFragment, - Config: Merge + Clone, - RoleConfig: Default + JsonSchema + Serialize, - CommonConfig: Default + JsonSchema + Serialize, - ConfigOverrides: Default + JsonSchema + Serialize, - { - let mut role_config = role.config.config.clone(); - role_config.merge(default_config); - let mut rolegroup_config = self.config.config.clone(); - rolegroup_config.merge(&role_config); - fragment::validate(rolegroup_config) - } -} - -/// A reference to a named role group of a given cluster object -#[derive(Educe)] -#[educe(Clone, Debug)] -pub struct RoleGroupRef { - pub cluster: ObjectRef, - pub role: String, - pub role_group: String, -} - -impl RoleGroupRef { - pub fn object_name(&self) -> String { - format!("{}-{}-{}", self.cluster.name, self.role, self.role_group) - } - - /// Returns the service name used by rolegroups for cluster internal communication only. - /// - /// The internal use of of this service name is indicated by the `-headless` suffix. - /// This service should not be used for communication to external services or clients - /// and also should not be used to export metrics (like Prometheus). Metrics should be - /// instead exposed via a dedicated service. Use [`Self::rolegroup_metrics_service_name`] - /// instead. - pub fn rolegroup_headless_service_name(&self) -> String { - format!("{name}-headless", name = self.object_name()) - } - - /// Returns the service name used by rolegroups to expose metrics (like Prometheus). - /// - /// The use for metrics only is indicated by the `-metrics` suffix. This service - /// should not be used for any internal communication or any other external - /// communication. For internal communication, use [`Self::rolegroup_headless_service_name`] - /// instead. - pub fn rolegroup_metrics_service_name(&self) -> String { - format!("{name}-metrics", name = self.object_name()) - } -} - -impl Display for RoleGroupRef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_fmt(format_args!( - "role group {}/{} of {}", - self.role, self.role_group, self.cluster - )) - } -} - -/// Returns [`Some`] in case the number of replicas is hard-coded to a certain value. -/// -/// This is the case when all `replicas` are set to [`Some`], in which case they are simply -/// summed. -/// -/// The argument `zero_replicas_counting` is a safety mechanism, which allows the caller to decide -/// if an explicit replica count of `0` should be treated as [`None`]. It also means that [`None`] -/// is returned in case no roleGroups are configured at all. -// -// Note: We are using a [`IntoIterator`] combined with `.peekable()` over [`ExactSizeIterator`] to -// have minimal bound requirements on the caller. -pub fn fixed_replica_count>>( - replicas: I, - zero_replicas_counting: ZeroReplicasCounting, -) -> Option { - let mut replicas = replicas.into_iter().peekable(); - - // An empty role has no fixed replica count when zeros are treated as None. - if zero_replicas_counting == ZeroReplicasCounting::TreatAsNone && replicas.peek().is_none() { - return None; - } - - replicas - .map(|replicas| match replicas { - None => None, - Some(0) if zero_replicas_counting == ZeroReplicasCounting::TreatAsNone => None, - // The individual replicas are [`u16`]s, so a [`u32`] sum has plenty of space. - Some(replicas) => Some(u32::from(replicas)), - }) - .sum() -} - -/// Returns the estimated total number of replicas across all role groups. -/// -/// Unlike [`fixed_replica_count`], this always returns a value: a role group with an unset (i.e. -/// [`None`]) replica count is assumed to run a single replica. Use this when a best-effort estimate -/// is needed even though the exact number of replicas is not hard-coded. -// -// Note: We are using a [`IntoIterator`] combined with `.peekable()` over [`ExactSizeIterator`] to -// have minimal bound requirements on the caller. -pub fn estimated_replica_count>>(replicas: I) -> u32 { - replicas - .into_iter() - .map(|replicas| u32::from(replicas.unwrap_or(1))) - .sum() -} - -/// How explicit zero (`0`) replicas on a role group should be counted -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum ZeroReplicasCounting { - /// Treat them as what they are: `Some(0)`. - TreatAsZero, - /// Treat them as if the user configured [`None`]. - TreatAsNone, -} - -#[cfg(test)] -mod tests { - use std::collections::HashSet; - - use super::*; - use crate::role_utils::JavaCommonConfig; - - #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] - struct EmptyConfigOverrides {} - - #[test] - fn test_merge_java_common_config() { - // The operator generates some JVM arguments - let operator_generated = JvmArgumentOverrides::new_with_only_additions( - [ - "-Xms34406m".to_owned(), - "-Xmx34406m".to_owned(), - "-XX:+UseG1GC".to_owned(), - "-XX:+ExitOnOutOfMemoryError".to_owned(), - "-Djava.protocol.handler.pkgs=sun.net.www.protocol".to_owned(), - "-Dsun.net.http.allowRestrictedHeaders=true".to_owned(), - "-Djava.security.properties=/stackable/nifi/conf/security.properties".to_owned(), - ] - .into(), - ); - - let entire_role: Role<(), EmptyConfigOverrides, GenericRoleConfig, JavaCommonConfig> = - serde_yaml::from_str(" - # Let's say we want to set some additional HTTP Proxy and IPv4 settings - # And we don't like the garbage collector for some reason... - jvmArgumentOverrides: - remove: - - -XX:+UseG1GC - add: # Add some networking arguments - - -Dhttps.proxyHost=proxy.my.corp - - -Dhttps.proxyPort=8080 - - -Djava.net.preferIPv4Stack=true - roleGroups: - default: - # For the roleGroup, let's say we need a different memory config. - # For that to work we first remove the flags generated by the operator and add our own. - # Also we override the proxy port to test that the roleGroup config takes precedence over the role config. - jvmArgumentOverrides: - removeRegex: - - -Xmx.* - - -Dhttps.proxyPort=.* - add: - - -Xmx40000m - - -Dhttps.proxyPort=1234 - ") - .expect("Failed to parse role"); - - let merged_jvm_argument_overrides = entire_role - .get_merged_jvm_argument_overrides("default", &operator_generated) - .expect("Failed to merge jvm argument overrides"); - - let expected = Vec::from([ - "-Xms34406m".to_owned(), - "-XX:+ExitOnOutOfMemoryError".to_owned(), - "-Djava.protocol.handler.pkgs=sun.net.www.protocol".to_owned(), - "-Dsun.net.http.allowRestrictedHeaders=true".to_owned(), - "-Djava.security.properties=/stackable/nifi/conf/security.properties".to_owned(), - "-Dhttps.proxyHost=proxy.my.corp".to_owned(), - "-Djava.net.preferIPv4Stack=true".to_owned(), - "-Xmx40000m".to_owned(), - "-Dhttps.proxyPort=1234".to_owned(), - ]); - - assert_eq!( - merged_jvm_argument_overrides, - JvmArgumentOverrides { - add: expected.clone(), - remove: HashSet::new(), - remove_regex: Vec::new() - } - ); - - assert_eq!( - merged_jvm_argument_overrides.effective_jvm_config_after_merging(), - &expected - ); - } - - #[test] - fn test_merge_java_common_config_keep_order() { - let operator_generated = - JvmArgumentOverrides::new_with_only_additions(["-Xms1m".to_owned()].into()); - - let entire_role: Role<(), EmptyConfigOverrides, GenericRoleConfig, JavaCommonConfig> = - serde_yaml::from_str( - " - jvmArgumentOverrides: - add: - - -Xms2m - roleGroups: - default: - jvmArgumentOverrides: - add: - - -Xms3m - ", - ) - .expect("Failed to parse role"); - - let merged_jvm_argument_overrides = entire_role - .get_merged_jvm_argument_overrides("default", &operator_generated) - .expect("Failed to merge jvm argument overrides"); - - assert_eq!( - merged_jvm_argument_overrides.effective_jvm_config_after_merging(), - &[ - "-Xms1m".to_owned(), - "-Xms2m".to_owned(), - "-Xms3m".to_owned() - ] - ); - } - - #[test] - fn replica_counts_with_all_replicas_set() { - let replicas = [Some(3), Some(2), Some(5)]; - - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), - Some(10) - ); - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), - Some(10) - ); - assert_eq!(estimated_replica_count(replicas), 10); - } - - #[test] - fn replica_counts_with_one_replica_unset() { - let replicas = [Some(3), None, Some(2)]; - - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), - None - ); - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), - None - ); - assert_eq!(estimated_replica_count(replicas), 6); - } - - #[test] - fn replica_counts_with_a_zero_replica() { - let replicas = [Some(3), Some(0)]; - - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), - Some(3) - ); - // With treat_zero_as_none the zero turns the whole count into None. - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), - None - ); - assert_eq!(estimated_replica_count(replicas), 3); - } - - #[test] - fn replica_counts_with_a_single_zero_role_groups_group() { - let replicas = [Some(0)]; - - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), - Some(0) - ); - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), - None - ); - assert_eq!(estimated_replica_count(replicas), 0); - } - - #[test] - fn replica_counts_without_role_groups() { - let replicas = []; - - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsZero), - Some(0) - ); - assert_eq!( - fixed_replica_count(replicas, ZeroReplicasCounting::TreatAsNone), - None - ); - assert_eq!(estimated_replica_count(replicas), 0); - } -} diff --git a/crates/stackable-operator/src/test_utils.rs b/crates/stackable-operator/src/test_utils.rs index 7c98a45ea..7c34f9d42 100644 --- a/crates/stackable-operator/src/test_utils.rs +++ b/crates/stackable-operator/src/test_utils.rs @@ -15,3 +15,14 @@ where )) }) } + +/// Deserialize a YAML string with recursive singleton maps instead of expecting custom tags. +pub fn deserialize_from_yaml_with_singleton_map<'a, T>( + input: &'a str, +) -> Result +where + T: serde::Deserialize<'a>, +{ + let deserializer = serde_yaml::Deserializer::from_str(input); + serde_yaml::with::singleton_map_recursive::deserialize(deserializer) +} diff --git a/crates/stackable-operator/src/v2/jvm_argument_overrides.rs b/crates/stackable-operator/src/v2/jvm_argument_overrides.rs index 821a9d7e9..0101980ed 100644 --- a/crates/stackable-operator/src/v2/jvm_argument_overrides.rs +++ b/crates/stackable-operator/src/v2/jvm_argument_overrides.rs @@ -124,7 +124,7 @@ mod tests { use super::*; use crate::{ - role_utils::GenericRoleConfig, + crd::role::GenericRoleConfig, v2::role_utils::{JavaCommonConfig, Role, RoleGroup, with_validated_config}, }; diff --git a/crates/stackable-operator/src/v2/role_utils.rs b/crates/stackable-operator/src/v2/role_utils.rs index fd55eb765..2e45b7ada 100644 --- a/crates/stackable-operator/src/v2/role_utils.rs +++ b/crates/stackable-operator/src/v2/role_utils.rs @@ -18,8 +18,8 @@ use crate::{ fragment::{self, FromFragment}, merge::{self, Merge, merge}, }, + crd::role::GenericRoleConfig, k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, - role_utils::GenericRoleConfig, schemars::{self, JsonSchema}, utils::crds::raw_object_schema, v2::env_overrides::EnvOverrides, @@ -213,11 +213,13 @@ pub struct JavaCommonConfig { pub jvm_argument_overrides: JvmArgumentOverrides, } -/// Variant of [`crate::role_utils::RoleGroup`] that is easier to work with +/// Variant of [`RoleGroup`][role] that is easier to work with /// /// Differences are: /// * `config` is flattened. /// * The [`EnvOverrides`] in `env_overrides` is replaced with an [`EnvVarSet`]. +/// +/// [role]: crate::crd::role #[derive(Clone, Debug, PartialEq)] pub struct RoleGroupConfig { pub replicas: Option, @@ -385,9 +387,9 @@ mod tests { use super::*; use crate::{ config::{fragment::Fragment, merge::Merge}, + crd::role::GenericRoleConfig, k8s_openapi::api::core::v1::PodTemplateSpec, kube::api::ObjectMeta, - role_utils::GenericRoleConfig, schemars::{self, JsonSchema}, v2::{ builder::pod::container::EnvVarName, diff --git a/crates/xtask/src/crd/dummy.rs b/crates/xtask/src/crd/dummy.rs index 67cbbfe11..c60d8d185 100644 --- a/crates/xtask/src/crd/dummy.rs +++ b/crates/xtask/src/crd/dummy.rs @@ -8,6 +8,7 @@ use stackable_operator::{ crd::{ authentication::{self, oidc}, git_sync::v1alpha2::GitSync, + role, }, database_connections::{ databases::{ @@ -21,7 +22,6 @@ use stackable_operator::{ }, deep_merger::ObjectOverrides, kube::CustomResource, - role_utils::Role, schemars::JsonSchema, status::condition::ClusterCondition, v2::config_overrides::JsonConfigOverrides, @@ -89,7 +89,7 @@ pub mod versioned { #[schemars(crate = "stackable_operator::schemars")] #[serde(rename_all = "camelCase")] pub struct DummyClusterSpec { - nodes: Option>, + nodes: Option>, // Not versioned yet stackable_affinity: stackable_operator::commons::affinity::StackableAffinity, From 56a753a5ee4c384b48f1c54ff4f9de5a725b9914 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 12 Oct 2026 11:17:54 +0200 Subject: [PATCH 2/2] chore(operator): Comment out "managed" variant from replicas --- crates/stackable-operator/crds/DummyCluster.yaml | 5 ----- crates/stackable-operator/src/crd/role/mod.rs | 13 +++++++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/stackable-operator/crds/DummyCluster.yaml b/crates/stackable-operator/crds/DummyCluster.yaml index 9c5289d68..33ff8616b 100644 --- a/crates/stackable-operator/crds/DummyCluster.yaml +++ b/crates/stackable-operator/crds/DummyCluster.yaml @@ -1750,8 +1750,6 @@ spec: oneOf: - required: - fixed - - required: - - managed - required: - custom properties: @@ -1769,9 +1767,6 @@ spec: required: - count type: object - managed: - description: Use a fully managed, auto-scaled number of replicas. - type: object type: object required: - replicas diff --git a/crates/stackable-operator/src/crd/role/mod.rs b/crates/stackable-operator/src/crd/role/mod.rs index 362baa8d3..dff8f175f 100644 --- a/crates/stackable-operator/src/crd/role/mod.rs +++ b/crates/stackable-operator/src/crd/role/mod.rs @@ -288,9 +288,11 @@ pub enum Replicas { // with the shape of an object AND primitive types, like a u16 in this case. Fixed { count: u16 }, - /// Use a fully managed, auto-scaled number of replicas. - Managed {}, - + // NOTE (@Techassi): Commented out for now, because fully managed autoscaling is tricky and will + // only come later. + // /// Use a fully managed, auto-scaled number of replicas. + // Managed {}, + // /// Use a auto-scaled number of replicas managed by external/custom mechanisms, e.g. HPA. Custom {}, } @@ -308,7 +310,10 @@ impl From for Option { fn from(replicas: Replicas) -> Self { match replicas { Replicas::Fixed { count } => Some(count), - Replicas::Managed {} => None, + // NOTE (@Techassi): Commented out for now, because fully managed autoscaling is tricky and will + // only come later. + // Replicas::Managed {} => None, + // FIXME (@Techassi): Deal with this (we need fallible conversions or we need to support // tracking type changes). Alternatively, we could map this to None as well, but that // would be a lossy conversion.