From 6a7ecbcfc0bf251e7495913a414e0e10486b2f27 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Wed, 4 Nov 2026 03:33:50 +0100 Subject: [PATCH] fix(cli): admin-less deploy installs on one node, in any namespace A namespace account could not finish a first install: Apicurio was refused on the kafka namespace, one missing PodMonitor grant switched off the compliance Role on clusters with no Prometheus operator, and Kafka's chart defaults left brokers Pending for 15 minutes with no reason given. Apicurio now follows the deploy's namespaces and cluster. The compliance Role's PodMonitor rule and the CLI's check for it follow the API the cluster serves. Admin-less detects kind, k3d and minikube without cluster rights, and sizes down to one broker where the stack would not fit, and --ha=false now sizes it. Readiness waits print why pods are Pending, and stop early when every node is short and nothing changes for three minutes. --- .github/workflows/ci.yml | 5 + charts/kates-platform/Chart.yaml | 5 +- charts/kates/Chart.yaml | 5 +- charts/kates/README.md | 5 +- charts/kates/templates/rbac.yaml | 6 +- charts/kates/values.yaml | 4 +- cli/cmd/cluster_gate_test.go | 4 +- cli/cmd/deploy.go | 45 +++- cli/cmd/deploy_adminless.go | 185 ++++++++++++++- cli/cmd/deploy_adminless_test.go | 210 +++++++++++++++++- cli/cmd/deploy_components.go | 82 ++++++- cli/cmd/deploy_components_test.go | 41 ++++ cli/cmd/silent_wait.go | 201 ++++++++++++++++- cli/cmd/silent_wait_test.go | 190 ++++++++++++++++ docs/book/12-deployment.md | 15 +- docs/book/appendix-d-versions.md | 4 +- .../kates/compliance/ComplianceCheck.java | 4 +- .../kates/service/ClusterTopologyService.java | 14 +- .../kates/compliance/ComplianceCheckTest.java | 60 +++++ .../service/ClusterTopologyServiceTest.java | 47 ++++ 20 files changed, 1086 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5450c9659..c82fc45c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -335,6 +335,7 @@ jobs: fail "a cluster-scoped object is rendered" fi rbac=$(helm template kates charts/kates/ -n team --set rbac.scope=namespace \ + --api-versions monitoring.coreos.com/v1/PodMonitor \ --set networkPolicy.kafka.namespace=kafka-ns --show-only templates/rbac.yaml) [ "$(grep -c '^ namespace: kafka-ns$' <<<"${rbac}")" = 2 ] \ || fail "the Role and RoleBinding are not both in networkPolicy.kafka.namespace" @@ -348,6 +349,10 @@ jobs: if grep -E '^ +verbs:' <<<"${rbac}" | grep -vqE 'verbs: \["(get|list)"\]$'; then fail "the Kafka-namespace Role grants more than get and list" fi + rbac=$(helm template kates charts/kates/ -n team --set rbac.scope=namespace \ + --set networkPolicy.kafka.namespace=kafka-ns --show-only templates/rbac.yaml) + grep -q 'podmonitors' <<<"${rbac}" && fail "the Role names PodMonitors where the API is not served" + [ "$(grep -c -- '- apiGroups:' <<<"${rbac}")" = 4 ] || fail "the Kafka-namespace Role has other than its 4 rules without PodMonitors" out=$(helm template kates charts/kates/ --set rbac.scope=namespace \ --set rbac.kafkaNamespaceRole=false --show-only templates/rbac.yaml 2>&1) || true grep -q 'could not find template' <<<"${out}" || fail "rbac.kafkaNamespaceRole=false still renders the Role" diff --git a/charts/kates-platform/Chart.yaml b/charts/kates-platform/Chart.yaml index d2165dcf4..1a610a46b 100644 --- a/charts/kates-platform/Chart.yaml +++ b/charts/kates-platform/Chart.yaml @@ -6,7 +6,8 @@ type: application # Patch: the umbrella packs the subcharts the v1.36.0 release pins: kates # 0.15.7, kafka-cluster 1.0.29, connect-cluster 2.1.24, mirror-maker2 0.11.24. # Patch: the chart icon; the subcharts it packs change only in theirs. -version: 0.8.23 +# Patch: packs kates 0.15.9. +version: 0.8.24 appVersion: "1.0.0" keywords: - kafka @@ -43,6 +44,8 @@ dependencies: condition: connect-cluster.enabled annotations: artifacthub.io/changes: | + - kind: fixed + description: "Packs kates 0.15.9, whose Kafka-namespace Role carries its podmonitors rule only where the PodMonitor API exists." - kind: added description: "A Kates chart icon. Packs kates 0.15.8, kafka-cluster 1.0.30, apicurio-registry 0.5.1, mirror-maker2 0.11.25 and connect-cluster 2.1.25, which change only in their icons. Templates are unchanged." - kind: changed diff --git a/charts/kates/Chart.yaml b/charts/kates/Chart.yaml index 495a7c0da..890dcd14a 100644 --- a/charts/kates/Chart.yaml +++ b/charts/kates/Chart.yaml @@ -5,7 +5,8 @@ type: application # Patch: appVersion and the image tag move to 1.36.0, the tag the v1.36.0 # release publishes. # Patch: the chart icon. -version: 0.15.8 +# Patch: the Kafka-namespace Role's PodMonitor rule follows the cluster. +version: 0.15.9 # The non-blocking health endpoints this chart's probes now point at come from # a BUILD-TIME property (quarkus.messaging.health.enabled=false), so they only # exist in the image built from this branch. Keep appVersion and @@ -29,6 +30,8 @@ annotations: artifacthub.io/category: integration-delivery artifacthub.io/license: Apache-2.0 artifacthub.io/changes: | + - kind: fixed + description: "With rbac.scope=namespace the Kafka-namespace Role carries its podmonitors rule only where the PodMonitor API exists, so a namespace admin can create it on a cluster without the Prometheus operator. helm template asks no cluster: pass --api-versions monitoring.coreos.com/v1/PodMonitor to render the rule for one that has it" - kind: fixed description: "The chart icon pointed at a file the repository does not have; it is now a Kates icon." - kind: changed diff --git a/charts/kates/README.md b/charts/kates/README.md index 4e4b20723..b57dd3ce0 100644 --- a/charts/kates/README.md +++ b/charts/kates/README.md @@ -114,13 +114,14 @@ that namespace (`kates/docs/deployment.md` runs one in `kates`) needs a | `kafka.strimzi.io` `kafkas` | `get` | | `kafka.strimzi.io` `kafkanodepools` | `list` | | `pods`, `persistentvolumeclaims` | `list` | -| `monitoring.coreos.com` `podmonitors` | `list` | +| `monitoring.coreos.com` `podmonitors`, where the cluster serves that API | `list` | | `discovery.k8s.io` `endpointslices` | `list` | -An installer with only `edit` in the release namespace can't create that Role, so it installs with `rbac.kafkaNamespaceRole=false` as well. Kubernetes lets an account create a Role only for permissions it holds itself, so an account with Strimzi's and the Prometheus Operator's reads in the Kafka namespace, a cluster admin where those operators don't add them to `admin`, renders it from the same release name, namespace and values, and applies it: +An installer with only `edit` in the release namespace can't create that Role, so it installs with `rbac.kafkaNamespaceRole=false` as well. Kubernetes lets an account create a Role only for permissions it holds itself, so an account with Strimzi's and the Prometheus Operator's reads in the Kafka namespace, a cluster admin where those operators don't add them to `admin`, renders it from the same release name, namespace and values, and applies it. `helm template` asks no cluster, so `--api-versions` keeps the PodMonitor rule; leave it out where the cluster has no Prometheus Operator: ```bash helm template kates oci://ghcr.io/bmscomp/charts/kates -n team -f team-values.yaml \ + --api-versions monitoring.coreos.com/v1/PodMonitor \ --set rbac.kafkaNamespaceRole=true --show-only templates/rbac.yaml | kubectl apply -f - ``` diff --git a/charts/kates/templates/rbac.yaml b/charts/kates/templates/rbac.yaml index ab6c2142e..2f265aa50 100644 --- a/charts/kates/templates/rbac.yaml +++ b/charts/kates/templates/rbac.yaml @@ -99,7 +99,9 @@ subjects: rbac.scope=namespace: nothing cluster-scoped. Performance and INTEGRITY runs make no Kubernetes call; a conformity check reads these in the Kafka namespace (ConformityCheck). It lists PodMonitors across namespaces first, - then here when that is refused. Nodes stay unread: zones, skew and + then here when that is refused; the rule renders only where the + PodMonitor API exists, so an account can create this Role on a cluster + without it. helm template needs --api-versions for it. Nodes stay unread: zones, skew and kafka.rackAwareness with source cr take each pod's zone from the EndpointSlices of Strimzi's headless Service, unless rbac.nodeReads grants them below. The release namespace ends @@ -125,9 +127,11 @@ rules: - apiGroups: [""] resources: ["pods", "persistentvolumeclaims"] verbs: ["list"] + {{- if .Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PodMonitor" }} - apiGroups: ["monitoring.coreos.com"] resources: ["podmonitors"] verbs: ["list"] + {{- end }} - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["list"] diff --git a/charts/kates/values.yaml b/charts/kates/values.yaml index 3b3f311ca..df7ce61c9 100644 --- a/charts/kates/values.yaml +++ b/charts/kates/values.yaml @@ -68,8 +68,8 @@ rbac: # -- With scope namespace: render the read-only Role and RoleBinding a # conformity check needs in the Kafka namespace (networkPolicy.kafka.namespace, # which the check then reads). An installer with only edit sets it false, and - # an account that may grant those reads applies them from - # `helm template ... --show-only templates/rbac.yaml`. + # an account that may grant those reads applies them from `helm template ... + # --api-versions monitoring.coreos.com/v1/PodMonitor --show-only templates/rbac.yaml`. kafkaNamespaceRole: true # -- With scope namespace: render a ClusterRole and ClusterRoleBinding that # grant get and list on nodes, so a conformity check reads any zoneLabel. diff --git a/cli/cmd/cluster_gate_test.go b/cli/cmd/cluster_gate_test.go index 4c5f1f369..f2d272df3 100644 --- a/cli/cmd/cluster_gate_test.go +++ b/cli/cmd/cluster_gate_test.go @@ -298,9 +298,9 @@ func setDeployFlags(t *testing.T, set func()) { kyverno, strimzi, connect := deployWithKyverno, deployWithStrimzi, deployWithKafkaConnect ui, mm2, dry := deployWithKafkaUI, deployWithMirrorMaker2, deployDryRun namespaceLabels := append([]string(nil), deployNamespaceLabels...) - adminless, dbNS := deployAdminless, deployDbNS + adminless, dbNS, ha := deployAdminless, deployDbNS, deployHA t.Cleanup(func() { - deployAdminless, deployDbNS = adminless, dbNS + deployAdminless, deployDbNS, deployHA = adminless, dbNS, ha deployTopology, deployNamespace, deployWithSchemaRegistry = topology, namespace, registry deployWithChaos, deployWithMonitoring, deployWithCertManager = chaos, monitoring, certManager deployWithKyverno, deployWithStrimzi, deployWithKafkaConnect = kyverno, strimzi, connect diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index a7c9230b5..635ab0a22 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -194,6 +194,12 @@ func runDeploy(cmd *cobra.Command, args []string) error { } // Deploy makes each namespace itself, or uses the one already there. defer dropCreateNamespace()() + + executor := defaultExecutor + report, err := adminlessReportFirst(cmd, executor, target, namespaceLabels) + if err != nil { + return err + } if err := configureDeploySelection(cmd, wizard); err != nil { return err } @@ -202,14 +208,15 @@ func runDeploy(cmd *cobra.Command, args []string) error { // go in before any pod, the pre-flight probes' included. A dry run loads // nothing. if !deployDryRun { - if err := loadBundleImages(context.Background(), target, quickDetectKind()); err != nil { + kubeContext, isKind := bundleKindTarget(target, report) + if err := loadBundleImages(context.Background(), kubeContext, isKind); err != nil { return err } } - executor := defaultExecutor - report, err := introspectDeployCluster(executor, target, namespaceLabels) - if err != nil { + if report != nil { + sizeAdminless(context.Background(), report, cmd.Flags().Changed("ha")) + } else if report, err = introspectDeployCluster(executor, target, namespaceLabels); err != nil { return err } @@ -476,13 +483,40 @@ func configureDeploySelection(cmd *cobra.Command, wizard bool) error { return nil } +// adminlessReportFirst reads the report before the wizard, admin-less only: it +// only reads, the wizard's HA question starts from the size this cluster takes, +// and it tells kind where quickDetectKind may not read. +func adminlessReportFirst(cmd *cobra.Command, executor detect.CommandExecutor, target string, namespaceLabels map[string]string) (*detect.DetectReport, error) { + if !deployAdminless { + return nil, nil + } + report, err := introspectDeployCluster(executor, target, namespaceLabels) + if err != nil { + return nil, err + } + defaultAdminlessSize(report, cmd.Flags().Changed("ha")) + return report, nil +} + +// bundleKindTarget says which context the bundle's images load through and +// whether it is kind: the admin-less report's when it told kind, since that +// account may not read quickDetectKind's DaemonSet, and its context carries +// kind's own name, where a team's context may not. +func bundleKindTarget(target string, report *detect.DetectReport) (string, bool) { + if report != nil && isKindReport(report) { + return report.Context, true + } + return target, quickDetectKind() +} + // introspectDeployCluster runs the pre-flight and the introspection that // sizes the values file. Admin-less it runs neither. func introspectDeployCluster(executor detect.CommandExecutor, target string, namespaceLabels map[string]string) (*detect.DetectReport, error) { if deployAdminless { // Introspection reads nodes and kube-system, and its probes create // namespaces and pods: nothing a namespace account may do. The charts' - // own defaults stand in for what it would have sized. + // own defaults stand in for what it would have sized, or the admin-less + // sizing shrinks them. // shortcut: assumes the cluster.local DNS domain; read it from a pod in the namespace if a cluster needs another. report := &detect.DetectReport{Context: target, Provider: "unknown"} report.Network.ClusterDomain = "cluster.local" @@ -491,6 +525,7 @@ func introspectDeployCluster(executor detect.CommandExecutor, target string, nam collector := detect.NewCollector(executor) collector.KubeContext = target _ = collector.CollectNodes(context.Background(), report) + detectAdminlessPlatform(context.Background(), report) return report, nil } PrintPhaseHeader(nextDeployPhase(), "Running Cluster Introspection (Pre-flight)") diff --git a/cli/cmd/deploy_adminless.go b/cli/cmd/deploy_adminless.go index 25bb11716..3e450fc4a 100644 --- a/cli/cmd/deploy_adminless.go +++ b/cli/cmd/deploy_adminless.go @@ -5,11 +5,13 @@ import ( "fmt" "maps" "slices" + "strconv" "strings" "github.com/charmbracelet/huh" "github.com/spf13/cobra" + "github.com/bmscomp/kates/cli/pkg/detect" "github.com/bmscomp/kates/cli/pkg/strimzi" ) @@ -106,6 +108,152 @@ func applyAdminlessProfile(ctx context.Context, cmd *cobra.Command) error { return nil } +// detectAdminlessPlatform tells kind, k3d and minikube with no cluster rights: +// from the kube context, the kubeconfig's cluster name, which each writes its +// own way whatever a team's context is called (the gate's fanon on kind-fanon), +// or the node's providerID when nodes were read. Only kind has overlays. +func detectAdminlessPlatform(ctx context.Context, report *detect.DetectReport) { + out, _ := runExecOutputFn(ctx, "kubectl", "config", "view", "--minify", "-o", "jsonpath={.clusters[0].name}") + cluster := strings.TrimSpace(string(out)) + for _, name := range []string{report.Context, cluster} { + switch { + case strings.HasPrefix(name, "k3d-"): + report.Provider = "k3d" + case name == "minikube": + report.Provider = "minikube" + } + } + if strings.HasPrefix(cluster, "kind-") { + // kindClusterName reads it from here: the context may be a team's. + report.Context = cluster + } + if strings.HasPrefix(report.Context, "kind-") || + slices.ContainsFunc(report.Nodes, func(n detect.NodeInfo) bool { return strings.HasPrefix(n.ProviderID, "kind://") }) { + report.Provider = "kind" + } +} + +// localCluster says the report names a kind, k3d or minikube cluster, whose +// nodes have no zones to count on even when they cannot be read. +func localCluster(report *detect.DetectReport) bool { + return isKindReport(report) || report.Provider == "k3d" || report.Provider == "minikube" +} + +// adminlessSmallCluster names what keeps the chart's three brokers Pending +// until the timeout, or says "": kind, a local cluster whose nodes are unread, +// or one node. Nodes it could not read on another cluster keep the chart's size. +func adminlessSmallCluster(report *detect.DetectReport) string { + switch { + case isKindReport(report): + return "a kind cluster" + case localCluster(report) && len(report.Nodes) == 0: + return "a " + report.Provider + " cluster" + case len(report.Nodes) == 1: + return "a single node" + } + return "" +} + +// defaultAdminlessSize turns HA off on a small cluster unless --ha was given, +// before the wizard, whose answer then stands. +func defaultAdminlessSize(report *detect.DetectReport, haGiven bool) { + if !haGiven && adminlessSmallCluster(report) != "" { + deployHA = false + } +} + +// sizeAdminless settles the size once the namespaces are known: a Kafka already +// there keeps its own, because a re-run that changes it would resize or strand +// that cluster, unless --ha was given. --ha=false is the way down. +func sizeAdminless(ctx context.Context, report *detect.DetectReport, haGiven bool) { + why := adminlessSmallCluster(report) + if why == "" { + why = "HA off" + } + if brokers := installedBrokers(ctx); brokers > 0 && !haGiven { + deployHA, why = brokers > 1, "the installed Kafka" + } + if oneBroker() { + PrintPhaseItem(fmt.Sprintf("Size: 1 controller, 1 broker, 1 Connect and MirrorMaker 2 worker, replication factor 1 (%s)", why)) + } +} + +// installedBrokers counts the brokers of the deploy's Kafka, 0 when there is +// none or the read fails. +func installedBrokers(ctx context.Context) int { + out, err := runExecOutputFn(ctx, "kubectl", "get", "kafkanodepools", "-n", resolveNamespaces().kafka, + "-l", "strimzi.io/cluster="+deployKafkaName, + "-o", `jsonpath={range .items[*]}{.spec.replicas}{" "}{.spec.roles}{"\n"}{end}`) + if err != nil { + return 0 + } + brokers := 0 + for _, pool := range strings.Split(string(out), "\n") { + replicas, roles, _ := strings.Cut(pool, " ") + if n, err := strconv.Atoi(replicas); err == nil && strings.Contains(roles, "broker") { + brokers += n + } + } + return brokers +} + +// adminlessServiceArgs keeps a Service off the kind overlays' fixed NodePorts, +// which are cluster-wide: another stack on the cluster would hold them. +func adminlessServiceArgs() []string { return []string{"--set", "service.type=ClusterIP"} } + +// oneBroker says the deploy runs the chart's own pools at one broker. Only +// admin-less does: the full profile's generated pools keep their count and +// names, because a pool renamed by a re-run is a pool deleted with its data. +func oneBroker() bool { return deployAdminless && !deployHA } + +// What one broker needs from each chart. Replicas sit in roleDefaults because +// the chart's default pools set none. Kafka's own topics follow the broker +// count; every other replication factor here would refuse or starve one broker. +var ( + oneBrokerKafkaArgs = []string{ + "--set", "nodePools.roleDefaults.controller.replicas=1", + "--set", "nodePools.roleDefaults.controller.resources.requests.cpu=250m", + "--set", "nodePools.roleDefaults.broker.replicas=1", + "--set", "nodePools.roleDefaults.broker.resources.requests.cpu=250m", + "--set", "nodePools.roleDefaults.broker.resources.requests.memory=1Gi", + "--set", "nodePools.roleDefaults.broker.resources.limits.memory=1Gi", + "--set", "nodePools.roleDefaults.broker.jvmOptions.-Xms=512m", + "--set", "nodePools.roleDefaults.broker.jvmOptions.-Xmx=512m", + "--set", "nodePools.roleDefaults.broker.storage.volumes[0].id=0", + "--set", "nodePools.roleDefaults.broker.storage.volumes[0].size=10Gi", + "--set", "kafka.config.default\\.replication\\.factor=1", + "--set", "kafka.config.min\\.insync\\.replicas=1", + "--set", "kafka.config.offsets\\.topic\\.replication\\.factor=1", + "--set", "kafka.config.transaction\\.state\\.log\\.replication\\.factor=1", + "--set", "kafka.config.transaction\\.state\\.log\\.min\\.isr=1", + "--set", "kafka.config.share\\.coordinator\\.state\\.topic\\.replication\\.factor=1", + "--set", "kafka.config.share\\.coordinator\\.state\\.topic\\.min\\.isr=1", + "--set", "cruiseControl.enabled=false", + "--set", "rebalance.enabled=false", + "--set", "kafkaExporter.enabled=false", + } + oneBrokerConnectArgs = []string{ + "--set", "config.replicationFactor=1", + "--set", "resources.requests.cpu=250m", + "--set", "resources.requests.memory=1Gi", + "--set", "jvmOptions.-Xms=512m", + "--set", "jvmOptions.-Xmx=512m", + } + // mirrors[0] by index merges into the overlay's list, which restates it. + oneBrokerMM2Args = []string{ + "--set", "target.config.config\\.storage\\.replication\\.factor=1", + "--set", "target.config.offset\\.storage\\.replication\\.factor=1", + "--set", "target.config.status\\.storage\\.replication\\.factor=1", + "--set", "mirrors[0].sourceConnector.config.replication\\.factor=1", + "--set", "mirrors[0].sourceConnector.config.offset-syncs\\.topic\\.replication\\.factor=1", + "--set", "mirrors[0].checkpointConnector.config.checkpoints\\.topic\\.replication\\.factor=1", + "--set", "resources.requests.cpu=250m", + "--set", "resources.requests.memory=512Mi", + "--set", "jvmOptions.-Xms=256m", + "--set", "jvmOptions.-Xmx=256m", + } +) + // dropCreateNamespace takes --create-namespace out of every helm call until // the returned function restores the seam. Deploy creates its namespaces // before Helm runs (ensureDeployNamespace) and keeps one that already exists; @@ -292,11 +440,26 @@ func describeLeftOut(components []leftOutComponent) string { return strings.Join(lines, "\n") } +const podMonitorResource = "podmonitors.monitoring.coreos.com" + +// podMonitorsServed reads API discovery, which any authenticated account may. +func podMonitorsServed(ctx context.Context) (bool, error) { + out, err := runExecOutputFn(ctx, "kubectl", "api-resources", "--api-group=monitoring.coreos.com", "-o", "name") + return slices.Contains(strings.Fields(string(out)), podMonitorResource), err +} + // prometheusOperatorAPI says whether the cluster serves PodMonitors, which // admin-less monitoring creates for a Prometheus already there. func prometheusOperatorAPI(ctx context.Context) bool { - out, _ := runExecOutputFn(ctx, "kubectl", "api-resources", "--api-group=monitoring.coreos.com", "-o", "name") - return slices.Contains(strings.Fields(string(out)), "podmonitors.monitoring.coreos.com") + served, _ := podMonitorsServed(ctx) + return served +} + +// podMonitorsAbsent says the cluster answered and serves no PodMonitors. A +// failed read is no such answer: the caller keeps asking for the grant. +func podMonitorsAbsent(ctx context.Context) bool { + served, err := podMonitorsServed(ctx) + return err == nil && !served } // describeStrimziOperator is one operator as the prerequisites list it. @@ -391,13 +554,14 @@ admin either sets strimzi-kafka-operator.createAggregateRoles=true on the Strimz // The reads the kates chart's Kafka-reader Role grants in the Kafka namespace // (charts/kates/templates/rbac.yaml), which a compliance check needs. RBAC lets // an account create a Role only for permissions it holds, so the install -// creates it only when the account holds all of them. +// creates it only when the account holds all of them. Where the cluster serves +// no PodMonitors the Role has no such rule, so the install does not ask for it. var complianceReaderGrants = [][2]string{ {"get", "kafkas.kafka.strimzi.io"}, {"list", "kafkanodepools.kafka.strimzi.io"}, {"list", "pods"}, {"list", "persistentvolumeclaims"}, - {"list", "podmonitors.monitoring.coreos.com"}, + {"list", podMonitorResource}, {"list", "endpointslices.discovery.k8s.io"}, } @@ -407,7 +571,11 @@ var adminlessComplianceMissing []string func missingComplianceGrants(ctx context.Context, kafkaNS string) []string { var missing []string + absent := podMonitorsAbsent(ctx) for _, g := range complianceReaderGrants { + if absent && g[1] == podMonitorResource { + continue + } answer, _ := runExecOutputFn(ctx, "kubectl", "auth", "can-i", g[0], g[1], "-n", kafkaNS) if strings.TrimSpace(string(answer)) != "yes" { missing = append(missing, g[0]+" "+g[1]) @@ -419,12 +587,17 @@ func missingComplianceGrants(ctx context.Context, kafkaNS string) []string { // complianceRoleHandoff tells the user how an admin turns compliance checks on // when the install had to leave the Role out. func complianceRoleHandoff(ns resolvedNamespaces, missing []string) string { + // helm template asks no cluster: without this it drops the PodMonitor rule. + apiVersions := "" + if !podMonitorsAbsent(context.Background()) { + apiVersions = " --api-versions monitoring.coreos.com/v1/PodMonitor" + } return fmt.Sprintf(`Compliance checks are off: Kates needs a read-only Role in namespace %s, and this account may not create it because it lacks: %s. Everything else works. To turn compliance checks on, an account that holds those reads there (a cluster admin) runs: - helm template kates %s -n %s --set rbac.scope=namespace \ + helm template kates %s -n %s --set rbac.scope=namespace%s \ --set networkPolicy.kafka.namespace=%s --show-only templates/rbac.yaml | kubectl apply -f -`, - ns.kafka, strings.Join(missing, ", "), chartRef("kates"), ns.app, ns.kafka) + ns.kafka, strings.Join(missing, ", "), chartRef("kates"), ns.app, apiVersions, ns.kafka) } diff --git a/cli/cmd/deploy_adminless_test.go b/cli/cmd/deploy_adminless_test.go index b2f52c779..995a391ba 100644 --- a/cli/cmd/deploy_adminless_test.go +++ b/cli/cmd/deploy_adminless_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/bmscomp/kates/cli/output" + "github.com/bmscomp/kates/cli/pkg/detect" "github.com/bmscomp/kates/cli/pkg/strimzi" "github.com/spf13/pflag" ) @@ -168,6 +169,191 @@ func TestRunDeploy_AdminlessInstallsTheStack(t *testing.T) { } } +func TestDetectAdminlessPlatform(t *testing.T) { + orig := runExecOutputFn + t.Cleanup(func() { runExecOutputFn = orig }) + + for _, tc := range []struct { + name, context, cluster string + nodes []detect.NodeInfo + kind bool + wantContext string + wantLocal bool + }{ + {"the gate's team context on kind", "fanon", "kind-fanon", nil, true, "kind-fanon", true}, + {"kind's own context", "kind-lab", "kind-lab", nil, true, "kind-lab", true}, + {"a kind node", "lab", "lab", []detect.NodeInfo{{ProviderID: "kind://docker/lab/lab-control-plane"}}, true, "lab", true}, + {"a shared cluster", "team", "prod-eu", []detect.NodeInfo{{ProviderID: "aws:///eu-west-1a/i-1"}}, false, "team", false}, + {"kind inside a name", "mankind", "mankind", nil, false, "mankind", false}, + {"a team's context on k3d", "team", "k3d-lab", nil, false, "team", true}, + {"minikube", "minikube", "minikube", nil, false, "minikube", true}, + } { + t.Run(tc.name, func(t *testing.T) { + runExecOutputFn = func(context.Context, string, ...string) ([]byte, error) { return []byte(tc.cluster + "\n"), nil } + report := &detect.DetectReport{Context: tc.context, Provider: "unknown", Nodes: tc.nodes} + detectAdminlessPlatform(context.Background(), report) + if isKindReport(report) != tc.kind || report.Context != tc.wantContext || localCluster(report) != tc.wantLocal { + t.Errorf("kind=%v context=%q local=%v, want %v %q %v", isKindReport(report), report.Context, localCluster(report), tc.kind, tc.wantContext, tc.wantLocal) + } + }) + } +} + +func TestSizeAdminless(t *testing.T) { + orig := runExecOutputFn + t.Cleanup(func() { runExecOutputFn = orig }) + one := []detect.NodeInfo{{Name: "n1"}} + for _, tc := range []struct { + name string + report detect.DetectReport + ha bool + haGiven bool + pools string + wantHA bool + }{ + {"kind with nodes refused", detect.DetectReport{Provider: "kind"}, true, false, "", false}, + {"k3d with nodes refused", detect.DetectReport{Provider: "k3d"}, true, false, "", false}, + {"k3d with several nodes", detect.DetectReport{Provider: "k3d", Nodes: append(one, detect.NodeInfo{Name: "n2"})}, true, false, "", true}, + {"one readable node", detect.DetectReport{Nodes: one}, true, false, "", false}, + {"several nodes", detect.DetectReport{Nodes: append(one, detect.NodeInfo{Name: "n2"})}, true, false, "", true}, + {"nodes refused, not kind", detect.DetectReport{}, true, false, "", true}, + {"--ha on one node is the user's", detect.DetectReport{Nodes: one}, true, true, "", true}, + {"--ha=false on several nodes", detect.DetectReport{Nodes: append(one, detect.NodeInfo{Name: "n2"})}, false, true, "", false}, + {"three brokers stay on kind", detect.DetectReport{Provider: "kind"}, true, false, "3 [\"controller\"]\n3 [\"broker\"]\n", true}, + {"one broker stays on several nodes", detect.DetectReport{Nodes: append(one, detect.NodeInfo{Name: "n2"})}, true, false, "1 [\"controller\",\"broker\"]\n", false}, + {"--ha over one broker is the user's", detect.DetectReport{Provider: "kind"}, true, true, "1 [\"broker\"]\n", true}, + } { + t.Run(tc.name, func(t *testing.T) { + setDeployFlags(t, func() { deployAdminless, deployHA = true, tc.ha }) + runExecOutputFn = func(context.Context, string, ...string) ([]byte, error) { return []byte(tc.pools), nil } + defaultAdminlessSize(&tc.report, tc.haGiven) + sizeAdminless(context.Background(), &tc.report, tc.haGiven) + if deployHA != tc.wantHA { + t.Errorf("deployHA = %v, want %v", deployHA, tc.wantHA) + } + }) + } +} + +// The wizard runs between the two: its yes on kind is not undone by the second. +func TestSizeAdminless_WizardYesStands(t *testing.T) { + orig := runExecOutputFn + t.Cleanup(func() { runExecOutputFn = orig }) + runExecOutputFn = func(context.Context, string, ...string) ([]byte, error) { return nil, nil } + setDeployFlags(t, func() { deployAdminless, deployHA = true, true }) + report := &detect.DetectReport{Provider: "kind"} + + defaultAdminlessSize(report, false) + if deployHA { + t.Fatal("the wizard's question does not start from one broker on kind") + } + deployHA = true + sizeAdminless(context.Background(), report, false) + if !deployHA { + t.Fatal("the wizard's yes was overridden") + } +} + +func TestBundleKindTarget(t *testing.T) { + orig := runExecOutputFn + t.Cleanup(func() { runExecOutputFn = orig }) + runExecOutputFn = func(context.Context, string, ...string) ([]byte, error) { + return nil, errors.New("daemonsets.apps is forbidden") + } + + if got, isKind := bundleKindTarget("fanon", &detect.DetectReport{Context: "kind-fanon", Provider: "kind"}); got != "kind-fanon" || !isKind { + t.Errorf("got %q %v, want kind's own context", got, isKind) + } + if got, isKind := bundleKindTarget("team", &detect.DetectReport{Context: "team", Provider: "unknown"}); got != "team" || isKind { + t.Errorf("got %q %v, want the target, not kind", got, isKind) + } + if got, isKind := bundleKindTarget("team", nil); got != "team" || isKind { + t.Errorf("full profile got %q %v", got, isKind) + } +} + +// The chart's rails refuse a replication factor above the brokers, so one +// broker must reach every chart: each of these keeps the deploy from failing, +// or hanging, on one broker. +func TestRunDeploy_AdminlessOnKindSizesEveryChart(t *testing.T) { + origOutput, origCombined, origHelm, origExec := runExecOutputFn, runExecCombinedFn, runHelmFn, runExecFn + origStdin, origDeployed := runExecStdinFn, isHelmReleaseDeployedFn + t.Cleanup(func() { + runExecOutputFn, runExecCombinedFn, runHelmFn, runExecFn = origOutput, origCombined, origHelm, origExec + runExecStdinFn, isHelmReleaseDeployedFn = origStdin, origDeployed + }) + setDeployFlags(t, func() { + deployAdminless, deployWithSchemaRegistry, deployDryRun, deployHA = true, "apicurio", false, true + deployWithChaos, deployWithMonitoring, deployWithCertManager, deployWithStrimzi = true, true, true, true + deployWithKafkaConnect, deployWithMirrorMaker2, deployWithKafkaUI = true, true, true + deployTopology = "isolated" + }) + setAdminlessNamespaces(t) + resetDeployFlagsChanged(t) + t.Chdir(t.TempDir()) + + read := adminlessClusterRead("") + stub := func(ctx context.Context, name string, args ...string) ([]byte, error) { + joined := name + " " + strings.Join(args, " ") + switch { + case strings.Contains(joined, "config view --minify -o jsonpath={.clusters[0].name}"): + return []byte("kind-fanon"), nil + case name == "docker" || name == "kind": + return nil, errors.New("no local image") + } + return read(ctx, name, args...) + } + runExecOutputFn, runExecCombinedFn = stub, stub + var mu sync.Mutex + installed := map[string]string{} + runHelmFn = func(_ context.Context, args ...string) error { + if len(args) > 2 && args[0] == "upgrade" { + mu.Lock() + installed[args[2]] = strings.Join(args, " ") + mu.Unlock() + } + return nil + } + runExecFn = func(context.Context, string, ...string) error { return nil } + runExecStdinFn = func(context.Context, string, []string, string) error { return nil } + isHelmReleaseDeployedFn = func(context.Context, string, string) bool { return false } + + if err := runDeploy(deployCmd, nil); err != nil { + t.Fatalf("runDeploy: %v", err) + } + for release, wants := range map[string][]string{ + "krafter": { + "charts/kafka-cluster/values-kind.yaml", + "--set nodePools.roleDefaults.broker.replicas=1", + "--set nodePools.roleDefaults.controller.replicas=1", + `--set kafka.config.offsets\.topic\.replication\.factor=1`, + "--set kafka.rack.topologyKey=kubernetes.io/hostname", + }, + "connect-cluster": { + "charts/connect-cluster/values-kind.yaml", "--set replicas=1", "--set config.replicationFactor=1", + "--set kafka.clusterName=krafter", + "--set networkPolicy.restApi.clients[0].namespace=team-kates", + "--set networkPolicy.restApi.clients[1].namespace=team-kafka", + "--set networkPolicy.restApi.clients[1].podSelector.app=kafka-ui", + }, + "mm2": {"--set replicas=1", `--set target.config.config\.storage\.replication\.factor=1`}, + "kates": {"--set-string defaults.replicationFactor=1", "--set-string defaults.minInsyncReplicas=1", "--set service.type=ClusterIP"}, + "kafka-ui": { + "http://connect-cluster-rest-api.team-connect.svc:8083", "apicurio-registry.team-kafka.svc:80", + "--set service.type=ClusterIP", "--set networkPolicy.extraEgress[0].ports[0].port=8080", + }, + } { + for _, want := range wants { + if !strings.Contains(installed[release], want) { + t.Errorf("%s: want %q in %s", release, want, installed[release]) + } + } + } + if c := installed["krafter"]; strings.Contains(c, "kafka.replicas") || strings.Contains(c, "zookeeper.replicas") { + t.Errorf("krafter sets keys the chart ignores: %s", c) + } +} + func TestCheckAdminlessPrerequisites(t *testing.T) { orig := runExecOutputFn t.Cleanup(func() { runExecOutputFn = orig }) @@ -230,10 +416,16 @@ func TestApplyAdminlessProfile_RefusesClusterScopedFlags(t *testing.T) { func TestRunDeploy_AdminlessComplianceRoleFollowsTheGrants(t *testing.T) { for _, tc := range []struct { name, refuse string + monitoring string // what discovery lists for monitoring.coreos.com + discovery error withRole bool + ruleInBundle bool // the admin's command renders the PodMonitor rule }{ - {"all reads held", "", true}, - {"no PodMonitor reads", "podmonitors", false}, + {"all reads held", "", podMonitorResource, nil, true, false}, + {"no PodMonitor reads", "podmonitors", podMonitorResource, nil, false, true}, + {"no PodMonitor reads, API not served", "podmonitors", "", nil, true, false}, + {"no PodMonitor reads, discovery failed", "podmonitors", "", errors.New("connection refused"), false, true}, + {"no pod reads, API not served", "can-i list pods", "", nil, false, false}, } { t.Run(tc.name, func(t *testing.T) { origOutput, origCombined, origHelm, origExec := runExecOutputFn, runExecCombinedFn, runHelmFn, runExecFn @@ -250,7 +442,14 @@ func TestRunDeploy_AdminlessComplianceRoleFollowsTheGrants(t *testing.T) { resetDeployFlagsChanged(t) t.Chdir(t.TempDir()) - runExecOutputFn, runExecCombinedFn = adminlessClusterRead(tc.refuse), adminlessClusterRead(tc.refuse) + read := adminlessClusterRead(tc.refuse) + runExecOutputFn = func(ctx context.Context, name string, args ...string) ([]byte, error) { + if slices.Contains(args, "--api-group=monitoring.coreos.com") { + return []byte(tc.monitoring), tc.discovery + } + return read(ctx, name, args...) + } + runExecCombinedFn = read var mu sync.Mutex var kates string runHelmFn = func(_ context.Context, args ...string) error { @@ -280,9 +479,12 @@ func TestRunDeploy_AdminlessComplianceRoleFollowsTheGrants(t *testing.T) { if printed := strings.Contains(out, handoff); printed == tc.withRole { t.Errorf("admin command printed = %v, want %v:\n%s", printed, !tc.withRole, out) } - if !tc.withRole && !strings.Contains(out, "list podmonitors.monitoring.coreos.com") { + if tc.refuse == "podmonitors" && !tc.withRole && !strings.Contains(out, "list "+podMonitorResource) { t.Errorf("the missing read is not named:\n%s", out) } + if got := strings.Contains(out, "--api-versions monitoring.coreos.com/v1/PodMonitor"); got != tc.ruleInBundle { + t.Errorf("admin command renders the PodMonitor rule = %v, want %v:\n%s", got, tc.ruleInBundle, out) + } }) } } diff --git a/cli/cmd/deploy_components.go b/cli/cmd/deploy_components.go index 23f6f41e8..de31d72cb 100644 --- a/cli/cmd/deploy_components.go +++ b/cli/cmd/deploy_components.go @@ -501,6 +501,7 @@ func isKafkaClusterReady(ctx context.Context, namespace, name string) bool { // cluster label too, and brokers run before they have joined. A paused // Kafka is not waited on: the operator will not reconcile it. func waitKafkaReady(ctx context.Context, namespace, name string, deadline time.Time) error { + pods := podWaitWhy{namespace: namespace, selector: "strimzi.io/cluster=" + name} for { r := readKafkaReadiness(ctx, namespace, name) switch { @@ -514,7 +515,10 @@ func waitKafkaReady(ctx context.Context, namespace, name string, deadline time.T if r.why != "" { why = " (" + r.why + ")" } - return fmt.Errorf("timeout waiting for Kafka %s/%s to report Ready for its current spec%s; is its operator watching namespace %s?", namespace, name, why, namespace) + return fmt.Errorf("timeout waiting for Kafka %s/%s to report Ready for its current spec%s%s; is its operator watching namespace %s?", namespace, name, why, pods.suffix(), namespace) + } + if err := pods.check(ctx); err != nil { + return err } select { case <-ctx.Done(): @@ -1200,19 +1204,18 @@ func (dc *deployContext) kafkaComponent() component { kafkaArgs = append(kafkaArgs, dc.versions.kafkaHelmArgs(dc.primary)...) } - if deployHA { + switch { + case oneBroker(): + kafkaArgs = append(kafkaArgs, oneBrokerKafkaArgs...) + case deployHA: kafkaArgs = append(kafkaArgs, - "--set", "kafka.replicas=3", "--set", "kafka.config.default\\.replication\\.factor=3", "--set", "kafka.config.min\\.insync\\.replicas=2", - "--set", "zookeeper.replicas=3", ) - } else { + default: kafkaArgs = append(kafkaArgs, - "--set", "kafka.replicas=1", "--set", "kafka.config.default\\.replication\\.factor=1", "--set", "kafka.config.min\\.insync\\.replicas=1", - "--set", "zookeeper.replicas=1", ) } @@ -1239,9 +1242,11 @@ func (dc *deployContext) kafkaComponent() component { // a zone, as generated values do: Strimzi makes rack a required node affinity on // brokers, which would stay Pending. It is for the admin-less deploy, which has // no generated values. Nodes it could not read keep the chart's zone key, right -// on the zoned clusters such accounts usually run on. +// on the zoned clusters such accounts usually run on, unless the cluster is a +// local one, which has none to count on, or there is one broker, which has no +// use for them. func (dc *deployContext) kafkaRackArgs() []string { - if len(dc.report.Nodes) > 0 && len(dc.report.Zones) == 0 { + if oneBroker() || len(dc.report.Zones) == 0 && (len(dc.report.Nodes) > 0 || localCluster(dc.report)) { return []string{"--set", "kafka.rack.topologyKey=kubernetes.io/hostname"} } return nil @@ -1420,12 +1425,18 @@ func (dc *deployContext) kafkaConnectHelmArgs(bootstrap, kafkaSvcDomain, connect "-f", dc.chartOverlay(chartRootPath("charts/connect-cluster")), "--set-string", "clusterDomain=" + connectDomain, "--set", "kafka.namespace=" + kafkaNS, + "--set", "kafka.clusterName=" + dc.primary.Name, "--set-string", "kafka.bootstrapServers=" + bootstrap, "--set", "schemaRegistry.enabled=true", "--set-string", "extraConfig.schema\\.registry\\.url=" + registryFQDN, "--set", "networkPolicy.egress.databases[0].namespace=" + deployDbNS, "--set", "networkPolicy.egress.databases[0].port=5432", "--set", "networkPolicy.egress.databases[0].podSelector.app\\.kubernetes\\.io/name=postgresql", + // A list given by index replaces the chart's, which names namespace kates. + "--set", "networkPolicy.restApi.clients[0].namespace=" + dc.ns.app, + "--set", "networkPolicy.restApi.clients[0].podSelector.app\\.kubernetes\\.io/name=kates", + "--set", "networkPolicy.restApi.clients[1].namespace=" + dc.ns.kafkaUI, + "--set", "networkPolicy.restApi.clients[1].podSelector.app=kafka-ui", "--timeout", "10m", } // Connect's spec.version must be inside the operator's window, so it @@ -1461,6 +1472,9 @@ func (dc *deployContext) kafkaConnectHelmArgs(bootstrap, kafkaSvcDomain, connect } else { connectArgs = append(connectArgs, "--set", "replicas=1") } + if oneBroker() { + connectArgs = append(connectArgs, oneBrokerConnectArgs...) + } // Monitoring is installed by now (Kafka installs after it), so the // CRDs its PodMonitor needs exist; without --with-monitoring the @@ -1604,12 +1618,31 @@ func (dc *deployContext) apicurioComponent() component { return fmt.Sprintf("\nšŸ“¦ Deploying Apicurio Schema Registry (Namespace: %s)...", kafkaNS) }, install: func(ctx context.Context) error { - return runHelmFn(ctx, append([]string{"upgrade", "--install", "apicurio", chartRootPath("charts/apicurio-registry"), "-n", kafkaNS, "--create-namespace", "--timeout", "5m"}, - dc.policyArgs("apicurio-registry")...)...) + return runHelmFn(ctx, dc.apicurioHelmArgs()...) }, } } +// apicurioHelmArgs is the apicurio release's argument list. Its namespace +// needs no --create-namespace: Kafka, a dependency, lives there. The chart's +// defaults name a cluster krafter in namespace kafka, and a list given by +// index replaces the chart's whole list, hence every client. +func (dc *deployContext) apicurioHelmArgs() []string { + kafkaNS := dc.ns.kafka + args := []string{"upgrade", "--install", "apicurio", chartRootPath("charts/apicurio-registry"), + "-n", kafkaNS, + "--set", "kafkaTopics.namespace=" + kafkaNS, + "--set", "kafkaTopics.clusterName=" + dc.primary.Name, + "--set-string", "global.kafka.bootstrapServers[0]=" + dc.primary.Bootstrap(dc.resolveClusterDomain()), + "--set", "networkPolicy.kafkaNamespace=" + kafkaNS, + "--set", "networkPolicy.clientNamespaces[0]=" + dc.ns.connect, + "--set", "networkPolicy.clientNamespaces[1]=" + dc.ns.app, + "--set", "networkPolicy.clientNamespaces[2]=" + dc.ns.kafkaUI, + "--timeout", "5m", + } + return append(args, dc.policyArgs("apicurio-registry")...) +} + // katesChaosArgs puts the Kates API on the noop chaos provider when this // deploy installs no Litmus and no chaos release is there. Left on its // litmus-crd default, the API finds no Litmus, logs an ERROR and falls back to @@ -1698,6 +1731,13 @@ func (dc *deployContext) katesComponent() component { if len(adminlessComplianceMissing) > 0 { katesArgs = append(katesArgs, "--set", "rbac.kafkaNamespaceRole=false") } + katesArgs = append(katesArgs, adminlessServiceArgs()...) + } + if oneBroker() { + katesArgs = append(katesArgs, + "--set-string", "defaults.replicationFactor=1", + "--set-string", "defaults.minInsyncReplicas=1", + ) } katesArgs = append(katesArgs, dc.policyArgs("kates")...) katesArgs = append(katesArgs, "--timeout", "8m") @@ -1743,6 +1783,20 @@ func (dc *deployContext) kafkaUIComponent() component { if overlay := dc.chartOverlay(chartRootPath("charts/kafka-ui")); dc.fileExists(overlay) { helmArgs = append(helmArgs, "-f", overlay) } + // values-kind.yaml's addresses name namespace kafka, and a Connect + // Service the chart no longer makes. The chart's egress takes the + // URL's port, 80, but NetworkPolicy sees the pod's. + if dc.isKind { + helmArgs = append(helmArgs, + "--set-string", "schemaRegistry.url=http://apicurio-apicurio-registry."+kafkaNS+".svc:80/apis/ccompat/v7", + "--set-string", "kafkaConnect.url=http://connect-cluster-rest-api."+dc.ns.connect+".svc:8083", + "--set", "networkPolicy.extraEgress[0].to[0].namespaceSelector.matchLabels.kubernetes\\.io/metadata\\.name="+kafkaNS, + "--set", "networkPolicy.extraEgress[0].ports[0].port=8080", + ) + } + if deployAdminless { + helmArgs = append(helmArgs, adminlessServiceArgs()...) + } return runHelmFn(ctx, append(helmArgs, dc.policyArgs("kafka-ui")...)...) }, } @@ -1798,6 +1852,12 @@ func (dc *deployContext) mirrorMaker2Component() component { "--set-string", "strimziVersion="+dc.versions.StrimziVersion, ) } + if !deployHA { + helmArgs = append(helmArgs, "--set", "replicas=1") + } + if oneBroker() { + helmArgs = append(helmArgs, oneBrokerMM2Args...) + } helmArgs = append(helmArgs, dc.scrapeArgs("charts/mirror-maker2")...) helmArgs = append(helmArgs, dc.policyArgs("mirror-maker2")...) helmArgs = append(helmArgs, "--timeout", "10m") diff --git a/cli/cmd/deploy_components_test.go b/cli/cmd/deploy_components_test.go index 1fc6f205c..245ed97aa 100644 --- a/cli/cmd/deploy_components_test.go +++ b/cli/cmd/deploy_components_test.go @@ -736,6 +736,44 @@ func TestDeployKafkaConnectStackReconcilesExistingReleaseWhenKafkaConnectNotRead } } +// Apicurio follows the deploy's Kafka namespace, cluster name and client +// namespaces; the chart's defaults are krafter in kafka, and an install in +// another namespace was refused on its KafkaTopics. +func TestApicurioHelmArgs_FollowTheDeploy(t *testing.T) { + dc := &deployContext{ + ns: resolvedNamespaces{kafka: "team-kafka", connect: "team-connect", app: "team-kates", kafkaUI: "team-ui"}, + primary: primaryCluster{Name: "warp", Namespace: "team-kafka"}, + report: &detect.DetectReport{}, + policyValues: map[string][]string{ + "apicurio-registry": {"-f", "policy.yaml"}, + }, + } + dc.report.Network.ClusterDomain = "taz.local" + + args := dc.apicurioHelmArgs() + cmd := strings.Join(args, " ") + for _, want := range []string{ + "-n team-kafka", + "--set kafkaTopics.namespace=team-kafka", + "--set kafkaTopics.clusterName=warp", + "--set-string global.kafka.bootstrapServers[0]=warp-kafka-bootstrap.team-kafka.svc.taz.local:9092", + "--set networkPolicy.kafkaNamespace=team-kafka", + "--set networkPolicy.clientNamespaces[0]=team-connect", + "--set networkPolicy.clientNamespaces[1]=team-kates", + "--set networkPolicy.clientNamespaces[2]=team-ui", + } { + if !strings.Contains(cmd, want) { + t.Errorf("Apicurio helm upgrade lacks %q:\n%s", want, cmd) + } + } + if strings.Contains(cmd, "--create-namespace") { + t.Errorf("Apicurio asks helm to create Kafka's namespace: %s", cmd) + } + if !strings.HasSuffix(cmd, "-f policy.yaml") { + t.Errorf("policy values are not last: %s", cmd) + } +} + // A Secret copy kubectl refuses fails the deploy there and then, naming the // Secret and the namespaces. The apply's error used to be dropped, and the // deploy went on to wait on a workload that could never start. The scheduler @@ -1019,6 +1057,9 @@ func TestKafkaRackArgs(t *testing.T) { {"nodes with no zone", detect.DetectReport{Nodes: nodes}, true}, {"nodes with zones", detect.DetectReport{Nodes: nodes, Zones: zones}, false}, {"nodes refused", detect.DetectReport{}, false}, + {"kind with nodes refused", detect.DetectReport{Provider: "kind"}, true}, + {"k3d with nodes refused", detect.DetectReport{Provider: "k3d"}, true}, + {"kind with zones", detect.DetectReport{Provider: "kind", Nodes: nodes, Zones: zones}, false}, } { t.Run(tc.name, func(t *testing.T) { dc := &deployContext{report: &tc.report} diff --git a/cli/cmd/silent_wait.go b/cli/cmd/silent_wait.go index 61819d39c..60a63adc2 100644 --- a/cli/cmd/silent_wait.go +++ b/cli/cmd/silent_wait.go @@ -2,7 +2,10 @@ package cmd import ( "context" + "encoding/json" "fmt" + "regexp" + "slices" "sort" "strings" "time" @@ -15,6 +18,13 @@ var ( silentHeartbeat = 30 * time.Second ) +// How often a wait reads why its pods are Pending, and how long the scheduler +// must say the same pods fit on no node before the wait gives up. +const ( + podBlockReadEvery = 15 * time.Second + podBlockHold = 3 * time.Minute +) + // waitComponentReadySilent polls pods matching a label selector until all are // Running/Succeeded or the timeout expires. It does NOT call // dl.StartComponent/FinishComponent ā€” the caller (deploy.go) is responsible @@ -23,6 +33,7 @@ func waitComponentReadySilent(ctx context.Context, namespace, selector string, t start := deployClock.Now() deadline := start.Add(timeout) lastBeat := start + why := podWaitWhy{namespace: namespace, selector: selector} for deployClock.Now().Before(deadline) { out, err := runExecOutputFn(ctx, "kubectl", "get", "pods", "-n", namespace, "-l", selector, "-o", "jsonpath={range .items[*]}{.status.phase}{','}{end}") var phases []string @@ -41,13 +52,195 @@ func waitComponentReadySilent(ctx context.Context, namespace, selector string, t dl.Printf(" %s\n", podHeartbeat(namespace, selector, phases, deployClock.Now().Sub(start))) } + if err := why.check(ctx); err != nil { + return err + } + select { case <-ctx.Done(): return ctx.Err() case <-deployClock.After(silentPollInterval): } } - return fmt.Errorf("timeout waiting for pods (selector=%s) to become ready", selector) + return fmt.Errorf("timeout waiting for pods (selector=%s) to become ready%s", selector, why.suffix()) +} + +// podWaitWhy says, during a readiness wait, why the wait's pods are Pending: +// each distinct reason once, and in the wait's error. A read that fails or is +// refused changes nothing, so the wait runs as if no one had asked. +type podWaitWhy struct { + namespace, selector string + nextRead time.Time + said map[string]bool + line string // the last read's reason, for the error + heldBy string // what has been blocking since heldSince + heldSince time.Time +} + +// check reads the reason when one is due. Its error stops the wait: every node +// has been short of CPU or memory for the same pods for podBlockHold, with no +// autoscaler adding one, which a longer wait will not change. +func (w *podWaitWhy) check(ctx context.Context) error { + now := deployClock.Now() + if now.Before(w.nextRead) { + return nil + } + w.nextRead = now.Add(podBlockReadEvery) + b, ok := readPodBlock(ctx, w.namespace, w.selector) + if !ok { + w.heldBy = "" + return nil + } + if b.detail == "" { + w.line, w.heldBy = "", "" + return nil + } + reason := b.problem + ": " + b.detail + w.line = fmt.Sprintf("%d pod(s) in %s %s", len(b.pods), w.namespace, reason) + if !w.said[reason] { + if w.said == nil { + w.said = map[string]bool{} + } + w.said[reason] = true + dl.Printf(" āš  %s\n", w.line) + } + heldBy := reason + " " + strings.Join(b.pods, " ") + switch { + case b.scalingUp || !everyNodeShort(b.detail): + w.heldBy = "" + case heldBy != w.heldBy: + w.heldBy, w.heldSince = heldBy, now + case now.Sub(w.heldSince) >= podBlockHold: + return fmt.Errorf("%s, unchanged for %s: stopped waiting", w.line, podBlockHold) + } + return nil +} + +// suffix is the last reason, for the end of a timeout error. +func (w *podWaitWhy) suffix() string { + if w.line == "" { + return "" + } + return ": " + w.line +} + +// podBlock is what keeps a wait's Pending pods from starting. An empty detail +// means nothing does, as far as the events say. +type podBlock struct { + problem, detail string + pods []string // the UIDs of the pods it holds back + scalingUp bool // an autoscaler answered for a Pending pod +} + +// podEvent is the part of a core/v1 Event a wait reads. The kubelet stamps +// lastTimestamp; the scheduler leaves it null and stamps eventTime and series. +type podEvent struct { + InvolvedObject struct { + UID string `json:"uid"` + } `json:"involvedObject"` + Reason string `json:"reason"` + Message string `json:"message"` + LastTimestamp time.Time `json:"lastTimestamp"` + EventTime time.Time `json:"eventTime"` + Series struct { + LastObservedTime time.Time `json:"lastObservedTime"` + } `json:"series"` +} + +func (e podEvent) at() time.Time { + t := e.LastTimestamp + for _, o := range []time.Time{e.EventTime, e.Series.LastObservedTime} { + if o.After(t) { + t = o + } + } + return t +} + +// readPodBlock reads the first reason a Pending pod of selector gives. Events +// are matched by UID, because a Strimzi pod comes back under its old name and +// an hour of the old pod's events remain; and a pod's newest event decides, so +// a FailedScheduling it has since got past does not count. ok is false when +// either read failed. +func readPodBlock(ctx context.Context, namespace, selector string) (b podBlock, ok bool) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + out, err := runExecOutputFn(ctx, "kubectl", "get", "pods", "-n", namespace, "-l", selector, "--field-selector=status.phase=Pending", "-o", "jsonpath={.items[*].metadata.uid}") + if err != nil { + return podBlock{}, false + } + uids := strings.Fields(string(out)) + if len(uids) == 0 { + return podBlock{}, true + } + out, err = runExecOutputFn(ctx, "kubectl", "get", "events", "-n", namespace, "--field-selector=involvedObject.kind=Pod", "-o", "json") + if err != nil { + return podBlock{}, false + } + var events struct { + Items []podEvent `json:"items"` + } + if err := json.Unmarshal(out, &events); err != nil { + return podBlock{}, false + } + slices.SortStableFunc(events.Items, func(x, y podEvent) int { return x.at().Compare(y.at()) }) + + newest := map[string]podEvent{} + pullError := map[string]string{} // the event that names the image and the cause + for _, e := range events.Items { + uid := e.InvolvedObject.UID + newest[uid] = e + if strings.Contains(e.Message, "Failed to pull image") { + pullError[uid] = e.Message + } + if slices.Contains(uids, uid) && (e.Reason == "TriggeredScaleUp" || e.Reason == "Nominated") { + b.scalingUp = true + } + } + for _, uid := range uids { + e := newest[uid] + problem, detail := "", "" + switch { + case e.Reason == "FailedScheduling": + problem = "cannot be scheduled" + detail, _, _ = strings.Cut(e.Message, " preemption:") + detail = strings.TrimRight(detail, ". ") + case imagePullFailed(e.Message): + problem, detail = "cannot pull an image", e.Message + if m := pullError[uid]; m != "" { + detail = m + } + default: + continue + } + detail = strings.Join(strings.Fields(detail), " ") + if b.detail == "" { + b.problem, b.detail = problem, detail + } + if problem == b.problem && detail == b.detail { + b.pods = append(b.pods, uid) + } + } + return b, true +} + +func imagePullFailed(message string) bool { + for _, s := range []string{"ImagePullBackOff", "ErrImagePull", "Failed to pull image", "Back-off pulling image"} { + if strings.Contains(message, s) { + return true + } + } + return false +} + +var nodesShort = regexp.MustCompile(`^0/\d+ nodes are available: (?:(?:, )?\d+ (?:Insufficient (?:cpu|memory)|node\(s\) had untolerated taint \{node-role\.kubernetes\.io/control-plane: \}))+$`) + +// everyNodeShort says the scheduler's message refuses every node for CPU or +// memory, or for the control-plane taint, which no wait clears either. A node +// refused for anything else (another taint, a volume, an affinity) is a reason +// this wait does not judge, so it does not stop on it. +func everyNodeShort(detail string) bool { + return nodesShort.MatchString(detail) && strings.Contains(detail, "Insufficient") } // podPhases splits kubectl's "Running,Pending," list, dropping the empty tail. @@ -130,18 +323,22 @@ func waitCustomResourceReadySilent(ctx context.Context, kind, namespace, selecto func waitNamedCustomResourceReadySilent(ctx context.Context, kind, namespace, name string, timeout time.Duration) error { deadline := deployClock.Now().Add(timeout) + pods := podWaitWhy{namespace: namespace, selector: "strimzi.io/cluster=" + name} for deployClock.Now().Before(deadline) { out, err := runExecOutputFn(ctx, "kubectl", "get", kind, name, "-n", namespace, "-o", `jsonpath={.status.conditions[?(@.type=="Ready")].status}`) if err == nil && strings.Contains(string(out), "True") { return nil } + if err := pods.check(ctx); err != nil { + return err + } select { case <-ctx.Done(): return ctx.Err() case <-deployClock.After(silentPollInterval): } } - return fmt.Errorf("timeout waiting for %s/%s to become ready", kind, name) + return fmt.Errorf("timeout waiting for %s/%s to become ready%s", kind, name, pods.suffix()) } func waitKafkaUsersReadySilent(ctx context.Context, namespace string, timeout time.Duration) error { diff --git a/cli/cmd/silent_wait_test.go b/cli/cmd/silent_wait_test.go index 893d62f1b..1fc01acfe 100644 --- a/cli/cmd/silent_wait_test.go +++ b/cli/cmd/silent_wait_test.go @@ -2,6 +2,8 @@ package cmd import ( "context" + "errors" + "fmt" "strings" "testing" "time" @@ -55,3 +57,191 @@ func TestPodHeartbeat(t *testing.T) { t.Errorf("podPhases: %q", got) } } + +// podEventsStub answers the wait's three reads: the phases, the Pending pods' +// UIDs and the namespace's events. A nil events read is refused. +func podEventsStub(t *testing.T, events func() []byte) { + t.Helper() + origOut, prevDL, prevClock := runExecOutputFn, dl, deployClock + t.Cleanup(func() { runExecOutputFn, dl, deployClock = origOut, prevDL, prevClock }) + dl, deployClock = nil, newFakeClock() + runExecOutputFn = func(_ context.Context, _ string, args ...string) ([]byte, error) { + all := strings.Join(args, " ") + switch { + case strings.Contains(all, "get events"): + if out := events(); out != nil { + return out, nil + } + return nil, errors.New("events is forbidden") + case strings.Contains(all, "status.phase=Pending"): + return []byte("u1 u2"), nil + } + return []byte("Running,Pending,Pending,"), nil + } +} + +const podEventsFormat = `{"items":[ + {"involvedObject":{"uid":"gone"},"reason":"FailedScheduling","message":"from the pod that had this name","eventTime":"2026-01-01T00:00:09.000000Z"}, + {"involvedObject":{"uid":"u1"},"reason":"FailedScheduling","message":"%[1]s","eventTime":"2026-01-01T00:00:01.000000Z","lastTimestamp":null,"series":null}, + {"involvedObject":{"uid":"u2"},"reason":"FailedScheduling","message":"%[1]s","eventTime":"2026-01-01T00:00:01.000000Z","series":{"lastObservedTime":"2026-01-01T00:00:05.000000Z"}} + %[2]s]}` + +const allShortOfCPU = "0/3 nodes are available: 3 Insufficient cpu. preemption: 0/3 nodes are available: 3 No preemption victims found for incoming pod." + +// The wait says why within a minute, once, and gives up when every node stays +// short of CPU, well before the timeout. +func TestWaitComponentReadySilent_NamesWhyAndStopsOnInsufficient(t *testing.T) { + podEventsStub(t, func() []byte { return fmt.Appendf(nil, podEventsFormat, allShortOfCPU, "") }) + + var err error + start := time.Time{} + out := captureStdout(t, func() { + start = deployClock.Now() + err = waitComponentReadySilent(context.Background(), "kafka", "strimzi.io/cluster=krafter", 15*time.Minute) + }) + want := "2 pod(s) in kafka cannot be scheduled: 0/3 nodes are available: 3 Insufficient cpu" + if strings.Count(out, want) != 1 { + t.Errorf("want the reason once:\n%s", out) + } + if strings.Contains(out, "from the pod that had this name") { + t.Errorf("an old pod's event was read as this one's:\n%s", out) + } + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("err = %v, want the reason", err) + } + if took := deployClock.Now().Sub(start); took < podBlockHold || took > 4*time.Minute { + t.Errorf("stopped after %s, want about %s", took, podBlockHold) + } +} + +func TestEveryNodeShort(t *testing.T) { + for detail, want := range map[string]bool{ + "0/3 nodes are available: 3 Insufficient cpu": true, + "0/1 nodes are available: 1 Insufficient cpu, 1 Insufficient memory": true, + "0/3 nodes are available: 2 Insufficient cpu, 1 Insufficient memory": true, + "0/4 nodes are available: 1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }, 3 Insufficient cpu": true, + "0/4 nodes are available: 1 node(s) had untolerated taint {a: b}, 3 Insufficient cpu": false, + "0/1 nodes are available: 1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }": false, + "0/3 nodes are available: 1 Insufficient cpu, 2 node(s) didn't match Pod's node affinity/selector": false, + "1/3 nodes are available: 2 Insufficient cpu": false, + } { + if got := everyNodeShort(detail); got != want { + t.Errorf("everyNodeShort(%q) = %v, want %v", detail, got, want) + } + } +} + +// The Kafka Connect wait reads its pods the way the others do: the workers can +// stay Pending where Kafka fit. +func TestWaitKafkaConnectReadySilent_StopsOnInsufficient(t *testing.T) { + podEventsStub(t, func() []byte { return fmt.Appendf(nil, podEventsFormat, allShortOfCPU, "") }) + + var err error + captureStdout(t, func() { + err = waitKafkaConnectReadySilent(context.Background(), "kafka", "connect-cluster", 15*time.Minute) + }) + if err == nil || !strings.Contains(err.Error(), "stopped waiting") || !strings.Contains(err.Error(), "3 Insufficient cpu") { + t.Errorf("err = %v, want the early stop naming the reason", err) + } +} + +// A wait that may yet clear runs to its timeout, and the timeout names the +// reason. +func TestWaitComponentReadySilent_NoEarlyStop(t *testing.T) { + scaleUp := `,{"involvedObject":{"uid":"u1"},"reason":"TriggeredScaleUp","message":"pod triggered scale-up","lastTimestamp":"2026-01-01T00:00:00Z"}` + imagePull := `,{"involvedObject":{"uid":"u1"},"reason":"Failed","message":"Failed to pull image \"kates:1\": not found","lastTimestamp":"2026-01-01T00:00:06Z"}, + {"involvedObject":{"uid":"u1"},"reason":"BackOff","message":"Back-off pulling image \"kates:1\"","lastTimestamp":"2026-01-01T00:00:07Z"}, + {"involvedObject":{"uid":"u2"},"reason":"BackOff","message":"Back-off pulling image \"kates:1\"","lastTimestamp":"2026-01-01T00:00:07Z"}` + for name, tc := range map[string]struct { + message, extra, want string + }{ + "an autoscaler is adding a node": {allShortOfCPU, scaleUp, "3 Insufficient cpu"}, + "some nodes refuse for a taint": {"0/4 nodes are available: 1 node(s) had untolerated taint {a: b}, 3 Insufficient cpu.", "", "1 node(s) had untolerated taint"}, + "some nodes refuse for a volume": {"0/3 nodes are available: 1 Insufficient cpu, 2 node(s) had volume node affinity conflict.", "", "2 node(s) had volume"}, + "only the control plane's taint": {"0/1 nodes are available: 1 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }.", "", "untolerated taint"}, + "an image will not pull": {allShortOfCPU, imagePull, `1 pod(s) in kafka cannot pull an image: Failed to pull image "kates:1": not found`}, + } { + t.Run(name, func(t *testing.T) { + podEventsStub(t, func() []byte { return fmt.Appendf(nil, podEventsFormat, tc.message, tc.extra) }) + var err error + start := time.Time{} + captureStdout(t, func() { + start = deployClock.Now() + err = waitComponentReadySilent(context.Background(), "kafka", "x=y", 15*time.Minute) + }) + if err == nil || !strings.Contains(err.Error(), "timeout waiting for pods") || !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want a timeout naming %q", err, tc.want) + } + if took := deployClock.Now().Sub(start); took < 15*time.Minute { + t.Errorf("stopped after %s, want the whole timeout", took) + } + }) + } +} + +// Pods that moved on leave no reason behind, and a pod scheduled since does +// not count as blocked. +func TestWaitComponentReadySilent_ReasonClears(t *testing.T) { + reads := 0 + podEventsStub(t, func() []byte { + reads++ + scheduled := "" + if reads > 1 { + for _, uid := range []string{"u1", "u2"} { + scheduled += fmt.Sprintf(`,{"involvedObject":{"uid":%q},"reason":"Scheduled","message":"assigned","eventTime":"2026-01-01T00:00:08.000000Z"}`, uid) + } + } + return fmt.Appendf(nil, podEventsFormat, allShortOfCPU, scheduled) + }) + var err error + out := captureStdout(t, func() { + err = waitComponentReadySilent(context.Background(), "kafka", "x=y", time.Minute) + }) + if !strings.Contains(out, "cannot be scheduled") { + t.Errorf("first read not reported:\n%s", out) + } + if err == nil || strings.Contains(err.Error(), "scheduled") { + t.Errorf("err = %v, want a plain timeout", err) + } +} + +// A refused events read is silent and leaves the wait as it was. +func TestWaitComponentReadySilent_EventsRefused(t *testing.T) { + for name, read := range map[string]func() []byte{ + "forbidden": func() []byte { return nil }, + "garbage": func() []byte { return []byte("not json") }, + } { + t.Run(name, func(t *testing.T) { + podEventsStub(t, read) + var err error + out := captureStdout(t, func() { + err = waitComponentReadySilent(context.Background(), "kafka", "x=y", 15*time.Minute) + }) + if want := "timeout waiting for pods (selector=x=y) to become ready"; err == nil || err.Error() != want { + t.Errorf("err = %v, want %q", err, want) + } + if strings.Contains(out, "āš ") || strings.Contains(out, "forbidden") { + t.Errorf("a refused read spoke:\n%s", out) + } + }) + } +} + +// Kafka's pods can go Pending after waitPods passed; the CR wait says so too. +func TestWaitKafkaReady_NamesWhyPodsWait(t *testing.T) { + podEventsStub(t, func() []byte { return fmt.Appendf(nil, podEventsFormat, allShortOfCPU, "") }) + inner := runExecOutputFn + runExecOutputFn = func(ctx context.Context, name string, args ...string) ([]byte, error) { + if strings.Contains(strings.Join(args, " "), "kafka krafter") { + return []byte("3|2|False||NotReady: brokers not up"), nil + } + return inner(ctx, name, args...) + } + var err error + captureStdout(t, func() { + err = waitKafkaReady(context.Background(), "kafka", "krafter", deployClock.Now().Add(15*time.Minute)) + }) + if err == nil || !strings.Contains(err.Error(), "3 Insufficient cpu") { + t.Errorf("err = %v, want the pods' reason", err) + } +} diff --git a/docs/book/12-deployment.md b/docs/book/12-deployment.md index 7f7870d27..520f99419 100644 --- a/docs/book/12-deployment.md +++ b/docs/book/12-deployment.md @@ -288,7 +288,7 @@ jvm: options: "-Xms512m -Xmx2560m -XX:+UseZGC -XX:+ZGenerational" ``` -These are the chart defaults, which a plain `helm install` runs with; `values-prod.yaml` sets the same memory and `jvm.options`, with a CPU limit of `1000m`. `kates deploy` runs the native image on every cluster, which `jvm.options` does not reach (see [Native Image Build](#native-image-build)). On Kind it applies `charts/kates/values-kind.yaml` and then `charts/kates/values-native-local.yaml`, a 384Mi request and a 512Mi limit. On any other cluster, admin-less installs included, it applies `charts/kates/values-generic.yaml` and then `charts/kates/values-native.yaml`, whose 768Mi request, 1Gi limit and CPU limit of 2 replace the generic overlay's JVM sizing. +These are the chart defaults, which a plain `helm install` runs with; `values-prod.yaml` sets the same memory and `jvm.options`, with a CPU limit of `1000m`. `kates deploy` runs the native image on every cluster, which `jvm.options` does not reach (see [Native Image Build](#native-image-build)). On Kind, admin-less installs included, it applies `charts/kates/values-kind.yaml` and then `charts/kates/values-native-local.yaml`, a 384Mi request and a 512Mi limit, when a local image is there. On any other cluster, admin-less installs included, it applies `charts/kates/values-generic.yaml` and then `charts/kates/values-native.yaml`, whose 768Mi request, 1Gi limit and CPU limit of 2 replace the generic overlay's JVM sizing. G1, the default, compares with the two low-pause collectors like this: @@ -1024,10 +1024,11 @@ The granting account creates a KafkaUser for the team's Kates, here `team-kates` - Create, Describe, Write and Read on the topic `kates-test-events`, and Read on the group `kates-webhooks`, which carry run events to webhooks; Describe and Read on the topic `kates-dlq`, which the Kates API polls, and Read on its group `kates-dlq-consumer`. - Describe and DescribeConfigs on every topic for a compliance check's `topics` rules, which judge only the topics the user may describe, and DescribeConfigs on the cluster for `kafka.source: brokers`. -It then renders the Role from the same release name, namespace and values as the team's install, so that it binds the right ServiceAccount, and applies it: +It then renders the Role from the same release name, namespace and values as the team's install, so that it binds the right ServiceAccount, and applies it. `helm template` asks no cluster, so `--api-versions` keeps the PodMonitor rule; leave it out where the cluster has no Prometheus Operator: ```bash helm template kates oci://ghcr.io/bmscomp/charts/kates -n team \ + --api-versions monitoring.coreos.com/v1/PodMonitor \ --set rbac.scope=namespace --show-only templates/rbac.yaml | kubectl apply -f - ``` @@ -1038,7 +1039,7 @@ The Role is `kates-kafka-reader-team` in the `kafka` namespace, and it only read | `kafkas` | get | The `Kafka` resource every check starts from | | `kafkanodepools` | list | Storage types and whether controllers are dedicated | | `pods`, `persistentvolumeclaims` | list | Counts, requests, limits, placement and storage sizes | -| `podmonitors` | list | Whether a PodMonitor selects the brokers | +| `podmonitors` | list | Whether a PodMonitor selects the brokers; rendered only where the cluster serves that API | | `endpointslices` | list | Each Kafka pod's zone, since nodes are cluster-scoped | Without the Role, a compliance check stops before its first rule: the Kates API answers 503 and says which grant is missing. @@ -1106,6 +1107,8 @@ kates deploy --adminless --topology single --namespace team It creates no namespace: each must exist, and the checks below name one the account can't use. It installs nothing cluster-scoped and skips the steps that need the cluster: Strimzi, cert-manager, Kyverno, the monitoring stack, Litmus, node introspection and namespace creation. `--with-chaos`, `--with-monitoring` and the other cluster-scoped toggles are refused with it. +Node introspection is skipped, so nothing sizes the stack to the cluster, and the charts' default (three controllers and three brokers, 100Gi each) needs several nodes. `--ha=false` installs one controller and one broker with a 10Gi disk, one Connect and one MirrorMaker 2 worker, replication factor 1 everywhere, and no Cruise Control or Kafka Exporter. Deploy picks that size itself on kind, k3d and minikube, told from the kube context or the kubeconfig's cluster name, and on any cluster whose nodes the account can read and that has one node; a k3d or minikube cluster whose nodes it can read and that has several keeps the default. A Kafka already in the namespace keeps its size on a re-run. Deploy prints the size and why; `--ha` or the wizard's answer keeps the default on a new install. On kind the charts' `values-kind.yaml` overlays apply too, with `ClusterIP` Services in place of their NodePorts, which are cluster-wide. With nodes the account can't read and that carry no zone label, on another cluster, set `kafka.rack.topologyKey: kubernetes.io/hostname` for `kafka-cluster` in [`.policies`](#kyverno-policies-file), or the three brokers stay `Pending` on Strimzi's required zone affinity. + Where the cluster serves the `monitoring.coreos.com` API, the wizard offers monitoring as the scrape alone. The Kafka, Connect, MirrorMaker 2 and Kates charts get their PodMonitors, ServiceMonitor and alert rules, for a Prometheus already on the cluster, whose selectors decide whether it reads these namespaces. The checks below then include the right to create PodMonitors, ServiceMonitors and PrometheusRules there. Before installing, it checks two things and stops with what is missing: @@ -1113,11 +1116,11 @@ Before installing, it checks two things and stops with what is missing: - A Strimzi operator is installed (the `kafka.strimzi.io` resources exist). A cluster admin installs it once, watching the Kafka, Connect and MirrorMaker 2 namespaces. - The account may create what the stack needs in each namespace: Strimzi's resources where each is made (the Kafka cluster, its topics and users in the Kafka namespace, Connect's in the Connect namespace, MirrorMaker 2's in its own), and everywhere Deployments, StatefulSets, Services, Secrets, ConfigMaps, ServiceAccounts, NetworkPolicies, Roles and RoleBindings. The built-in `admin` role covers all but Strimzi's. A cluster admin adds those by setting `strimzi-kafka-operator.createAggregateRoles=true` on the Strimzi release, which adds them to `admin` and `edit`. -Compliance checks need the read-only Role the `kates` chart creates in the Kafka namespace ([Step 1 ā€” Grant the Team's Kates Access to Kafka](#step-1--grant-the-teams-kates-access-to-kafka) lists its grants). Kubernetes lets an account create a Role only for permissions it holds, so `kates deploy --adminless` first asks whether the account holds each of them there. When it does, the Role is installed and compliance checks work. When it lacks any (PodMonitors are the usual one: the built-in `admin` role doesn't include them), Kates installs without it (`rbac.kafkaNamespaceRole=false`), and everything else works. Deploy then names the missing reads and prints the command an account that holds them runs to turn compliance checks on: +Compliance checks need the read-only Role the `kates` chart creates in the Kafka namespace ([Step 1 ā€” Grant the Team's Kates Access to Kafka](#step-1--grant-the-teams-kates-access-to-kafka) lists its grants). Kubernetes lets an account create a Role only for permissions it holds, so `kates deploy --adminless` first asks whether the account holds each of them there, PodMonitors only where the cluster serves that API. When it does, the Role is installed and compliance checks work. When it lacks any (PodMonitors are the usual one: the built-in `admin` role doesn't include them), Kates installs without it (`rbac.kafkaNamespaceRole=false`), and everything else works. Deploy then names the missing reads and prints the command an account that holds them runs to turn compliance checks on: ```bash helm template kates oci://ghcr.io/bmscomp/charts/kates -n team-kates \ - --set rbac.scope=namespace \ + --set rbac.scope=namespace --api-versions monitoring.coreos.com/v1/PodMonitor \ --set networkPolicy.kafka.namespace=team-kafka \ --show-only templates/rbac.yaml | kubectl apply -f - ``` @@ -1233,6 +1236,8 @@ The pods' events name the constraint the scheduler couldn't satisfy: kubectl describe pods -l strimzi.io/cluster=krafter -n kafka ``` +While it waits, `kates deploy` reads the events of its `Pending` pods and prints the first `FailedScheduling` or image pull reason, once per reason, and the failed wait's error repeats it. The wait for Kafka Connect does the same. If every node stays short of CPU or memory for the same pods for 3 minutes, the control plane's taint aside, and no autoscaler has answered for them, the wait stops there rather than running out its timeout. An account that may not read events sees none of this, and the wait runs as before. + Pods stuck in `Pending` mean the scheduler can't satisfy zone affinity or provision storage ā€” [Pods Stuck in Pending](20-installation-guide.md#pods-stuck-in-pending) in Installing Kafka with the kafka-cluster Helm Chart walks the `FailedScheduling` events constraint by constraint. Crashing brokers and a Kafka CR stuck on `NotReady` are diagnosed symptom by symptom in [Troubleshooting](15-kafka-deployment.md#troubleshooting) in Kafka Deployment Engineering. ### CLI Binary Killed on macOS diff --git a/docs/book/appendix-d-versions.md b/docs/book/appendix-d-versions.md index 49ae9875a..416df05aa 100644 --- a/docs/book/appendix-d-versions.md +++ b/docs/book/appendix-d-versions.md @@ -22,8 +22,8 @@ Every version the platform pins, in one place. This table is generated from the | `kafka-common` chart | 0.1.3 (library) | `charts/kafka-common/Chart.yaml` | | `kafka-ui` chart | 0.3.4 (app v1.5.0) | `charts/kafka-ui/Chart.yaml` | | `kates-chaos` chart | 2.4.3 (app 3.28.0) | `charts/kates-chaos/Chart.yaml` | -| `kates-platform` chart | 0.8.23 (app 1.0.0) | `charts/kates-platform/Chart.yaml` | -| `kates` chart | 0.15.8 (app 1.36.0) | `charts/kates/Chart.yaml` | +| `kates-platform` chart | 0.8.24 (app 1.0.0) | `charts/kates-platform/Chart.yaml` | +| `kates` chart | 0.15.9 (app 1.36.0) | `charts/kates/Chart.yaml` | | `legacy-kafka` chart | 0.2.2 (app 3.9.1) | `charts/legacy-kafka/Chart.yaml` | | `mirror-maker2` chart | 0.11.25 (app 4.3.1) | `charts/mirror-maker2/Chart.yaml` | | `monitoring` chart | 1.8.1 (app 82.4.3) | `charts/monitoring/Chart.yaml` | diff --git a/kates/src/main/java/com/bmscomp/kates/compliance/ComplianceCheck.java b/kates/src/main/java/com/bmscomp/kates/compliance/ComplianceCheck.java index da87eaf27..69b9c552a 100644 --- a/kates/src/main/java/com/bmscomp/kates/compliance/ComplianceCheck.java +++ b/kates/src/main/java/com/bmscomp/kates/compliance/ComplianceCheck.java @@ -276,7 +276,9 @@ private final class Run { .getItems(), true); } catch (KubernetesClientException e) { - if (e.getCode() == 404) { + // RBAC answers before routing: with no rule for it, a CRD that is not there is a 403. + if (e.getCode() == 404 + || e.getCode() == 403 && !client.supports("monitoring.coreos.com/v1", "PodMonitor")) { return new Monitors(null, true); } throw e; diff --git a/kates/src/main/java/com/bmscomp/kates/service/ClusterTopologyService.java b/kates/src/main/java/com/bmscomp/kates/service/ClusterTopologyService.java index 67d3cb455..5946c90de 100644 --- a/kates/src/main/java/com/bmscomp/kates/service/ClusterTopologyService.java +++ b/kates/src/main/java/com/bmscomp/kates/service/ClusterTopologyService.java @@ -558,7 +558,7 @@ private Map describeMetrics(Map kafkaCr, Map "True".equals(c.getStatus())); } + /** RBAC answers before routing: with no rule for it, a CRD that is not there is a 403, and no refused read. */ + private void refusedPodMonitors(Map unread, Exception e) { + try { + if (forbidden(e) == null || !kubernetesClient.supports("monitoring.coreos.com/v1", "PodMonitor")) { + return; + } + } catch (KubernetesClientException undiscovered) { + LOG.debug("Unable to discover PodMonitors", undiscovered); + } + refused(unread, "metrics", e); + } + public static class KubernetesNotAvailableException extends RuntimeException { public KubernetesNotAvailableException(String message) { super(message); diff --git a/kates/src/test/java/com/bmscomp/kates/compliance/ComplianceCheckTest.java b/kates/src/test/java/com/bmscomp/kates/compliance/ComplianceCheckTest.java index 185fd6bb1..e65739a4c 100644 --- a/kates/src/test/java/com/bmscomp/kates/compliance/ComplianceCheckTest.java +++ b/kates/src/test/java/com/bmscomp/kates/compliance/ComplianceCheckTest.java @@ -24,6 +24,7 @@ import jakarta.ws.rs.ClientErrorException; import jakarta.ws.rs.ServiceUnavailableException; +import io.fabric8.kubernetes.api.model.APIResourceListBuilder; import io.fabric8.kubernetes.api.model.AffinityBuilder; import io.fabric8.kubernetes.api.model.GenericKubernetesResource; import io.fabric8.kubernetes.api.model.GenericKubernetesResourceBuilder; @@ -977,6 +978,65 @@ void withoutThePodMonitorCrdThereIsNone() throws IOException { assertEquals("the PodMonitor CRD is not installed", result.detail()); } + /** The chart's Role has no PodMonitor rule where the API is missing, so RBAC refuses before the 404. */ + @Test + void aRefusalOfAnApiTheClusterDoesNotServeMeansThereIsNoCrd() throws IOException { + server.expect() + .get() + .withPath("/apis/monitoring.coreos.com/v1/podmonitors") + .andReturn(403, "") + .always(); + server.expect() + .get() + .withPath("/apis/monitoring.coreos.com/v1/namespaces/kafka/podmonitors") + .andReturn(403, "") + .always(); + server.expect() + .get() + .withPath("/apis/monitoring.coreos.com/v1") + .andReturn(404, "") + .always(); + + Result result = result(run("{monitoring: {podMonitor: true}}"), "monitoring.podMonitor"); + + assertEquals(Status.FAIL, result.status()); + assertEquals("the PodMonitor CRD is not installed", result.detail()); + } + + /** With the API served, the same refusal is a missing grant. */ + @Test + void aRefusalOfAnApiTheClusterServesStaysUnknown() throws IOException { + server.expect() + .get() + .withPath("/apis/monitoring.coreos.com/v1/podmonitors") + .andReturn(403, "") + .always(); + server.expect() + .get() + .withPath("/apis/monitoring.coreos.com/v1/namespaces/kafka/podmonitors") + .andReturn(403, "") + .always(); + server.expect() + .get() + .withPath("/apis/monitoring.coreos.com/v1") + .andReturn( + 200, + new APIResourceListBuilder() + .addNewResource() + .withKind("PodMonitor") + .withName("podmonitors") + .withNamespaced(true) + .withSingularName("podmonitor") + .withVerbs("list") + .endResource() + .build()) + .always(); + + Result result = result(run("{monitoring: {podMonitor: true}}"), "monitoring.podMonitor"); + + assertEquals(Status.UNKNOWN, result.status()); + } + /** A namespace-scoped role, such as the chart's rbac.scope=namespace, lists the Kafka namespace's only. */ @Test void withoutAccessToOtherNamespacesItReadsTheKafkaNamespacesPodMonitors() throws IOException { diff --git a/kates/src/test/java/com/bmscomp/kates/service/ClusterTopologyServiceTest.java b/kates/src/test/java/com/bmscomp/kates/service/ClusterTopologyServiceTest.java index c4d187635..2d0be9af4 100644 --- a/kates/src/test/java/com/bmscomp/kates/service/ClusterTopologyServiceTest.java +++ b/kates/src/test/java/com/bmscomp/kates/service/ClusterTopologyServiceTest.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; +import io.fabric8.kubernetes.api.model.APIResourceListBuilder; import io.fabric8.kubernetes.api.model.StatusBuilder; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient; @@ -88,6 +89,52 @@ void aRefusedPodListNamesTheGrantRatherThanKubernetes() { e.getMessage()); } + @Test + void aRefusedPodMonitorListOfAnApiNotServedLeavesNothingUnread() { + refuse( + "/apis/monitoring.coreos.com/v1/namespaces/kafka/podmonitors", + "list", + "podmonitors", + "monitoring.coreos.com"); + server.expect() + .get() + .withPath("/apis/monitoring.coreos.com/v1") + .andReturn(404, "") + .always(); + + assertEquals(Map.of(), service.describeTopology().get("unread")); + } + + @Test + @SuppressWarnings("unchecked") + void aRefusedPodMonitorListOfAnApiServedIsUnread() { + refuse( + "/apis/monitoring.coreos.com/v1/namespaces/kafka/podmonitors", + "list", + "podmonitors", + "monitoring.coreos.com"); + server.expect() + .get() + .withPath("/apis/monitoring.coreos.com/v1") + .andReturn( + 200, + new APIResourceListBuilder() + .addNewResource() + .withKind("PodMonitor") + .withName("podmonitors") + .withNamespaced(true) + .withSingularName("podmonitor") + .withVerbs("list") + .endResource() + .build()) + .always(); + + Map unread = + (Map) service.describeTopology().get("unread"); + + assertEquals(refusal("list", "podmonitors", "monitoring.coreos.com"), unread.get("metrics")); + } + void refuse(String path, String verb, String resource, String group) { server.expect() .get()