diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 403af343..3e370b4d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,6 +42,9 @@ jobs: - name: Unit tests run: make unit + - name: Helm template tests + run: make helm-test + - name: Vet run: make vet diff --git a/.gitignore b/.gitignore index 01dc05d8..c1d5cf8b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ chart/bootc-operator/charts/ chart/bootc-operator/Chart.lock chart/bootc-operator-crds/templates/ chart/bootc-operator/templates/controller-clusterrole.yaml +chart/bootc-operator/templates/controller-role.yaml +chart/bootc-operator/templates/controller-rolebinding.yaml chart/bootc-operator/templates/daemon-clusterrole.yaml chart/bootc-operator/templates/leader-election-role.yaml chart/bootc-operator/templates/controller-deployment.yaml diff --git a/Makefile b/Makefile index 52605716..f6a9f1a9 100644 --- a/Makefile +++ b/Makefile @@ -17,10 +17,14 @@ BINK_NODE_DISK_IMAGE ?= ghcr.io/bootc-dev/bink/node:v$(DEFAULT_KUBE_MINOR)-fedor BINK_LOCAL_REGISTRY_NODE_IMAGE ?= registry.cluster.local:5000/node E2E_REGISTRY_USER ?= e2e-user E2E_REGISTRY_PASSWORD ?= e2e-password -# Released operator for upgrade testing (TestOperatorUpgrade). -# Override RELEASED_OPERATOR_IMG for downstream images. +# Extra namespace deploy-bink allows pull secrets from (TestPullSecretNamespaces). +E2E_PULL_SECRET_NAMESPACE ?= bootc-e2e-pull-secrets +# Released operator for upgrade testing (TestOperatorUpgrade): the published +# chart and the release tag, whose version without the leading "v" is the +# chart version. The released image is the one the chart deploys. Override +# RELEASED_OPERATOR_CHART for downstream charts. RELEASED_OPERATOR_TAG ?= v0.2.2 -RELEASED_OPERATOR_IMG ?= ghcr.io/bootc-dev/bootc-operator:$(RELEASED_OPERATOR_TAG) +RELEASED_OPERATOR_CHART ?= oci://ghcr.io/bootc-dev/bootc-operator/charts/bootc-operator IMG_BINK_RELEASED ?= registry.cluster.local:5000/bootc-operator-released:latest # YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header. YEAR ?= $(shell date +%Y) @@ -62,6 +66,10 @@ manifests: controller-gen ## Generate CRD and RBAC manifests. helm: manifests yq ## Regenerate Helm chart from Kustomize manifests. YQ="$(YQ)" ./hack/sync-helm.sh +.PHONY: helm-test +helm-test: helm helm-tool ## Render the Helm chart and check the output. + HELM="$(HELM)" YQ="$(YQ)" ./hack/test-helm.sh + .PHONY: generate generate: controller-gen ## Generate DeepCopy method implementations. "$(CONTROLLER_GEN)" object:headerFile="hack/boilerplate.go.txt",year=$(YEAR) paths="./..." @@ -119,8 +127,10 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= E2E_NODE_IMAGE_UPDATE_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update | jq -r '.Digest') \ E2E_NODE_IMAGE_UPDATE2_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update2 | jq -r '.Digest') \ E2E_REGISTRY_USER=$(E2E_REGISTRY_USER) E2E_REGISTRY_PASSWORD=$(E2E_REGISTRY_PASSWORD) \ + E2E_PULL_SECRET_NAMESPACE=$(E2E_PULL_SECRET_NAMESPACE) \ $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \ - $(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \ + $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \ + $(if $(RELEASED_OPERATOR_CHART),E2E_OPERATOR_RELEASED_CHART=$(RELEASED_OPERATOR_CHART)) \ go test -timeout 90m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') . # EKS e2e settings @@ -152,7 +162,7 @@ e2e-eks: ## Run e2e tests against EKS. V=1 for verbose. RUN= to filter. E2E_REGISTRY_USER="$(EKS_REGISTRY_USER)" \ E2E_REGISTRY_PASSWORD="$(EKS_REGISTRY_PASSWORD)" \ $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \ - $(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(RELEASED_OPERATOR_IMG)) \ + $(if $(RELEASED_OPERATOR_CHART),E2E_OPERATOR_RELEASED_CHART=$(RELEASED_OPERATOR_CHART)) \ $(if $(EKS_IMAGE_PULL_SECRET),E2E_IMAGE_PULL_SECRET=$(EKS_IMAGE_PULL_SECRET)) \ E2E_NODE_PULL_SECRET="$(EKS_NODE_PULL_SECRET)" \ ARTIFACTS="$(ARTIFACTS)" \ @@ -192,13 +202,14 @@ build-update-image: ## Build derived node images for update testing and push to podman push --tls-verify=false localhost:5000/node:update2 .PHONY: push-released-operator-image -push-released-operator-image: ## Pull released operator image and push to bink registry for upgrade testing. - @if [ -z "$(RELEASED_OPERATOR_IMG)" ]; then \ - echo "Error: RELEASED_OPERATOR_IMG must be set (e.g. ghcr.io/bootc-dev/bootc-operator:v0.1.0)"; \ - exit 1; \ - fi - podman pull $(RELEASED_OPERATOR_IMG) - podman push --tls-verify=false $(RELEASED_OPERATOR_IMG) localhost:5000/bootc-operator-released:latest +push-released-operator-image: yq ## Copy the image of the released chart to the bink registry for upgrade testing. + img=$$(helm template bootc-operator $(RELEASED_OPERATOR_CHART) \ + --version $(RELEASED_OPERATOR_TAG:v%=%) \ + --kube-version $(DEFAULT_KUBE_MINOR).0 | \ + $(YQ) 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.name == "manager") | .image') && \ + test -n "$$img" && \ + podman pull "$$img" && \ + podman push --tls-verify=false "$$img" localhost:5000/bootc-operator-released:latest ##@ Deployment @@ -256,9 +267,11 @@ start-bink: seed-node-image ## Start a bink cluster (idempotent). kubectl --kubeconfig $(KUBECONFIG_BINK) wait --for=condition=Ready node/controller --timeout=5m .PHONY: deploy-bink -deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_IMG),push-released-operator-image) helm ## Deploy to a bink cluster (requires: buildimg). +deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_TAG),push-released-operator-image) helm ## Deploy to a bink cluster (requires: buildimg). podman push --tls-verify=false $(IMG) localhost:5000/bootc-operator-e2e:latest helm dependency build ./chart/bootc-operator + kubectl --kubeconfig $(KUBECONFIG_BINK) create namespace $(E2E_PULL_SECRET_NAMESPACE) \ + --dry-run=client -o yaml | kubectl --kubeconfig $(KUBECONFIG_BINK) apply -f - img="$(IMG_BINK)" && \ helm upgrade --install bootc-operator ./chart/bootc-operator \ --kubeconfig $(abspath $(KUBECONFIG_BINK)) \ @@ -266,6 +279,7 @@ deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_IMG),push-re --set-string image.repository="$${img%:*}" \ --set-string image.tag="$${img##*:}" \ --set-string image.pullPolicy=Always \ + --set-json 'pullSecretNamespaces=["$(E2E_PULL_SECRET_NAMESPACE)"]' \ --set-json 'controller.extraArgs=["--allow-insecure-registry","--tag-resolution-interval=10s"]' kubectl --kubeconfig $(KUBECONFIG_BINK) -n bootc-operator rollout restart deployment/bootc-operator-controller kubectl --kubeconfig $(KUBECONFIG_BINK) -n bootc-operator rollout status deployment/bootc-operator-controller --timeout=3m @@ -294,12 +308,14 @@ GOLANGCI_LINT = $(LOCALBIN)/golangci-lint GOLINES_BIN = $(LOCALBIN)/golines GOVULNCHECK = $(LOCALBIN)/govulncheck YQ ?= $(LOCALBIN)/yq +HELM ?= $(LOCALBIN)/helm KUSTOMIZE_VERSION ?= v5.8.1 CONTROLLER_TOOLS_VERSION ?= v0.20.1 GOLANGCI_LINT_VERSION ?= v2.11.4 GOVULNCHECK_VERSION ?= v1.9.0 YQ_VERSION ?= v4.53.2 +HELM_VERSION ?= v3.19.0 # ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) ENVTEST_VERSION ?= $(shell v='$(call gomodver,sigs.k8s.io/controller-runtime)'; \ @@ -353,6 +369,11 @@ yq: $(YQ) ## Download yq locally if necessary. $(YQ): $(LOCALBIN) $(call go-install-tool,$(YQ),github.com/mikefarah/yq/v4,$(YQ_VERSION)) +.PHONY: helm-tool +helm-tool: $(HELM) ## Download helm locally if necessary. +$(HELM): $(LOCALBIN) + $(call go-install-tool,$(HELM),helm.sh/helm/v3/cmd/helm,$(HELM_VERSION)) + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary # $2 - package url which can be installed diff --git a/chart/bootc-operator/templates/_helpers.tpl b/chart/bootc-operator/templates/_helpers.tpl index e16553e1..090b6cc8 100644 --- a/chart/bootc-operator/templates/_helpers.tpl +++ b/chart/bootc-operator/templates/_helpers.tpl @@ -23,6 +23,10 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} +{{- define "bootc-operator.pullSecretNamespaces" -}} +{{- prepend .Values.pullSecretNamespaces .Release.Namespace | uniq | join "," }} +{{- end }} + {{- define "bootc-operator.image" -}} {{- $tag := default .Chart.AppVersion .Values.image.tag -}} {{ printf "%s:%s" .Values.image.repository $tag }} diff --git a/chart/bootc-operator/templates/pull-secret-namespaces-rbac.yaml b/chart/bootc-operator/templates/pull-secret-namespaces-rbac.yaml new file mode 100644 index 00000000..8b3fc74a --- /dev/null +++ b/chart/bootc-operator/templates/pull-secret-namespaces-rbac.yaml @@ -0,0 +1,37 @@ +{{- range without (.Values.pullSecretNamespaces | uniq) $.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "bootc-operator.fullname" $ }}-pull-secrets + namespace: {{ . }} + labels: + {{- include "bootc-operator.labels" $ | nindent 4 }} + app.kubernetes.io/component: controller +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "bootc-operator.fullname" $ }}-pull-secrets + namespace: {{ . }} + labels: + {{- include "bootc-operator.labels" $ | nindent 4 }} + app.kubernetes.io/component: controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "bootc-operator.fullname" $ }}-pull-secrets +subjects: +- kind: ServiceAccount + name: {{ include "bootc-operator.fullname" $ }}-controller + namespace: {{ $.Release.Namespace }} +{{- end }} diff --git a/chart/bootc-operator/values.schema.json b/chart/bootc-operator/values.schema.json new file mode 100644 index 00000000..7c4847ef --- /dev/null +++ b/chart/bootc-operator/values.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "pullSecretNamespaces": { + "type": "array", + "items": { + "type": "string", + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + } + } + } +} diff --git a/chart/bootc-operator/values.yaml b/chart/bootc-operator/values.yaml index 383c3705..a91f91eb 100644 --- a/chart/bootc-operator/values.yaml +++ b/chart/bootc-operator/values.yaml @@ -10,5 +10,9 @@ image: imagePullSecrets: [] +# Extra namespaces that BootcNodePool pullSecretRefs may point to. The release +# namespace is always allowed. The namespaces must already exist. +pullSecretNamespaces: [] + controller: extraArgs: [] diff --git a/cmd/controller/main.go b/cmd/controller/main.go index f15223d3..f00f6931 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -49,6 +50,7 @@ func main() { var probeAddr string var tagResolutionInterval time.Duration var allowInsecureRegistry bool + var pullSecretNamespacesFlag string flag.StringVar( &probeAddr, "health-probe-bind-address", @@ -67,6 +69,13 @@ func main() { false, "Allow falling back to HTTP when resolving tag-based image refs against registries that do not serve TLS.", ) + flag.StringVar( + &pullSecretNamespacesFlag, + "pull-secret-namespaces", + "", + "Comma-separated extra namespaces that pools may reference pull secrets from. "+ + "The operator namespace is always allowed.", + ) flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -88,11 +97,29 @@ func main() { version.GitCommit, ) + operatorNamespace := detectNamespace() + if operatorNamespace == "" { + setupLog.Error(nil, "Operator namespace unknown; set POD_NAMESPACE") + os.Exit(1) + } + pullSecretNamespaces, err := controller.ParsePullSecretNamespaces( + pullSecretNamespacesFlag, + operatorNamespace, + ) + if err != nil { + setupLog.Error(err, "Invalid --pull-secret-namespaces") + os.Exit(1) + } + setupLog.Info("Pull secret namespaces", "namespaces", pullSecretNamespaces) + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "bootc-operator.bootc.dev", + Cache: cache.Options{ + ByObject: controller.SecretCacheByObject(pullSecretNamespaces), + }, }) if err != nil { setupLog.Error(err, "Failed to start manager") @@ -109,14 +136,24 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), KubeClient: kubeClient, - EventNamespace: detectNamespace(), + EventNamespace: operatorNamespace, TagResolver: ®istry.GGCRResolver{AllowInsecure: allowInsecureRegistry}, TagResolutionInterval: tagResolutionInterval, + PullSecretNamespaces: pullSecretNamespaces, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "bootcnodepool") os.Exit(1) } + if err := (&controller.PullSecretAggregator{ + Client: mgr.GetClient(), + Namespace: operatorNamespace, + PullSecretNamespaces: pullSecretNamespaces, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "Failed to create controller", "controller", "pullsecretaggregator") + os.Exit(1) + } + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index 829142f4..6d39660d 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -21,6 +21,7 @@ import ( bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" "github.com/bootc-dev/bootc-operator/internal/bootc" "github.com/bootc-dev/bootc-operator/internal/daemon" + "github.com/bootc-dev/bootc-operator/internal/pullsecret" "github.com/bootc-dev/bootc-operator/internal/version" ) @@ -106,9 +107,9 @@ func main() { Scheme: mgr.GetScheme(), NodeName: nodeName, HostRoot: "/proc/1/root", + PullSecretDir: pullsecret.MountDir, Executor: executor, StatusWatcher: watcher, - APIReader: mgr.GetAPIReader(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "bootcnode") os.Exit(1) diff --git a/config/daemon/daemon.yaml b/config/daemon/daemon.yaml index 7b8c8bc9..f701fdc3 100644 --- a/config/daemon/daemon.yaml +++ b/config/daemon/daemon.yaml @@ -42,6 +42,17 @@ spec: requests: cpu: 10m memory: 64Mi + volumeMounts: + # No subPath: subPath mounts do not receive Secret updates. + - name: pull-secrets + mountPath: /var/run/bootc-pull-secrets + readOnly: true + volumes: + - name: pull-secrets + secret: + secretName: bootc-pull-secrets + # The daemon must start before the controller creates the Secret. + optional: true tolerations: - operator: Exists serviceAccountName: daemon diff --git a/config/rbac/daemon_role.yaml b/config/rbac/daemon_role.yaml index f026a8eb..7e87c360 100644 --- a/config/rbac/daemon_role.yaml +++ b/config/rbac/daemon_role.yaml @@ -6,12 +6,6 @@ metadata: app.kubernetes.io/managed-by: kustomize name: daemon-role rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - apiGroups: - node.bootc.dev resources: diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 809aa028..066d954c 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -2,6 +2,7 @@ resources: - controller_service_account.yaml - role.yaml - role_binding.yaml +- namespaced_role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml - daemon_service_account.yaml diff --git a/config/rbac/namespaced_role_binding.yaml b/config/rbac/namespaced_role_binding.yaml new file mode 100644 index 00000000..a4b2f20b --- /dev/null +++ b/config/rbac/namespaced_role_binding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: bootc-operator + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding + namespace: system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c25de146..f6f17c53 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -27,14 +27,6 @@ rules: - pods/eviction verbs: - create -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - apiGroups: - "" - events.k8s.io @@ -82,3 +74,28 @@ rules: - bootcnodes/status verbs: - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: manager-role + namespace: system +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - get + - list + - watch +- apiGroups: + - "" + resourceNames: + - bootc-pull-secrets + resources: + - secrets + verbs: + - patch + - update diff --git a/docs/src/helm.md b/docs/src/helm.md index 91a73e49..141bb8c1 100644 --- a/docs/src/helm.md +++ b/docs/src/helm.md @@ -121,6 +121,15 @@ helm uninstall bootc-operator --namespace bootc-operator > **Warning**: uninstalling the chart also removes the CRDs and all > their instances cluster-wide. +The controller creates the `bootc-pull-secrets` Secret in the release +namespace, which Helm does not track, and `helm uninstall` does not +delete the namespace created by `--create-namespace`. Delete the namespace +to remove that Secret, along with any pull secrets created there: + +```bash +kubectl delete namespace bootc-operator +``` + ## Regenerating the chart Most Helm templates are generated from the kustomize manifests. After diff --git a/hack/sync-helm.sh b/hack/sync-helm.sh index f55df976..434b3102 100755 --- a/hack/sync-helm.sh +++ b/hack/sync-helm.sh @@ -20,10 +20,13 @@ echo "Synced CRDs" # --- RBAC --- # gen_rbac OUTPUT_FILE SOURCE_FILE KIND NAME_SUFFIX COMPONENT [NAMESPACE] +# Takes the rules from the KIND document in SOURCE_FILE, which may hold +# several (controller-gen emits namespaced rules as a separate Role). gen_rbac() { local output="$1" source="$2" kind="$3" suffix="$4" component="$5" ns="${6:-}" local rules - rules=$("$YQ" -I2 '.rules' "$source") + rules=$("$YQ" -I2 "select(.kind == \"${kind}\") | .rules" "$source") + [ -n "$rules" ] || { echo "error: no ${kind} rules in ${source}" >&2; exit 1; } { echo "apiVersion: rbac.authorization.k8s.io/v1" echo "kind: ${kind}" @@ -40,6 +43,8 @@ gen_rbac() { gen_rbac "$CHART_DIR/templates/controller-clusterrole.yaml" "$CONFIG_DIR/rbac/role.yaml" \ ClusterRole controller controller +gen_rbac "$CHART_DIR/templates/controller-role.yaml" "$CONFIG_DIR/rbac/role.yaml" \ + Role controller controller "{{ .Release.Namespace }}" gen_rbac "$CHART_DIR/templates/daemon-clusterrole.yaml" "$CONFIG_DIR/rbac/daemon_role.yaml" \ ClusterRole daemon daemon gen_rbac "$CHART_DIR/templates/leader-election-role.yaml" "$CONFIG_DIR/rbac/leader_election_role.yaml" \ @@ -84,8 +89,9 @@ gen_workload() { gen_workload "$CONFIG_DIR/manager/manager.yaml" Deployment controller \ "$CHART_DIR/templates/controller-deployment.yaml" -# Inject controller.extraArgs into the args list +# Inject --pull-secret-namespaces and controller.extraArgs into the args list sed -i '/^ image: {{ include "bootc-operator\.image" \. }}/i\ + - {{ printf "--pull-secret-namespaces=%s" (include "bootc-operator.pullSecretNamespaces" .) | quote }}\ {{- range .Values.controller.extraArgs }}\ - {{ . | quote }}\ {{- end }}' "$CHART_DIR/templates/controller-deployment.yaml" @@ -112,7 +118,8 @@ gen_binding() { .metadata.labels.HELM_LABELS_PLACEHOLDER = true | .roleRef.name = \"HELM_FULLNAME-${role_suffix}\" | .subjects[0].name = \"HELM_FULLNAME-${sa_suffix}\" | - .subjects[0].namespace = \"HELM_NAMESPACE\" + .subjects[0].namespace = \"HELM_NAMESPACE\" | + with(select(.metadata.namespace != null); .metadata.namespace = \"HELM_NAMESPACE\") " "$source" | helm_sed | sed \ -e '/HELM_LABELS_PLACEHOLDER/c\ {{- include "bootc-operator.labels" . | nindent 4 }}' \ > "$output" @@ -121,6 +128,9 @@ gen_binding() { gen_binding "$CONFIG_DIR/rbac/role_binding.yaml" \ "$CHART_DIR/templates/controller-clusterrolebinding.yaml" \ controller ClusterRole controller controller +gen_binding "$CONFIG_DIR/rbac/namespaced_role_binding.yaml" \ + "$CHART_DIR/templates/controller-rolebinding.yaml" \ + controller Role controller controller gen_binding "$CONFIG_DIR/rbac/daemon_role_binding.yaml" \ "$CHART_DIR/templates/daemon-clusterrolebinding.yaml" \ daemon ClusterRole daemon daemon diff --git a/hack/test-helm.sh b/hack/test-helm.sh new file mode 100755 index 00000000..c436eb69 --- /dev/null +++ b/hack/test-helm.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Render the Helm chart with representative values and check the output. +# Run: make helm-test + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +CHART_DIR="$REPO_ROOT/chart/bootc-operator" +HELM="${HELM:-helm}" +YQ="${YQ:-yq}" +NS=bootc-operator +SA=test-bootc-operator-controller + +fail() { echo "FAIL: $*" >&2; exit 1; } + +# A go-installed helm lacks build-time version info and assumes Kubernetes +# v1.20, which the chart's kubeVersion constraint rejects. +render() { + "$HELM" template test "$CHART_DIR" --namespace "$NS" --kube-version 1.34.0 "$@" +} + +# check MANIFESTS FLAG_VALUE NAMESPACES... +# Asserts that the controller flag is FLAG_VALUE, that each namespace has +# exactly one secret-reading Role bound to the controller ServiceAccount, +# and that no ClusterRole (controller or daemon) grants access to secrets. +check() { + local manifests="$1" flag="$2"; shift 2 + local want got + want=$(printf '%s\n' "$@" | sort) + + got=$(echo "$manifests" | "$YQ" -N 'select(.kind == "Deployment") | + .spec.template.spec.containers[0].args[] | select(test("^--pull-secret-namespaces="))') + [ "$got" = "--pull-secret-namespaces=$flag" ] || fail "controller flag is [$got]" + + roles=$(echo "$manifests" | "$YQ" -N 'select(.kind == "Role") | + select(.rules | any_c(.resources | any_c(. == "secrets"))) | + .metadata.namespace + "/" + .metadata.name' | sort) + [ "$(echo "$roles" | cut -d/ -f1)" = "$want" ] || fail "secret Roles: [$roles]" + + got=$(echo "$manifests" | "$YQ" -N 'select(.kind == "RoleBinding") | + select(.subjects | any_c(.name == "'"$SA"'" and .namespace == "'"$NS"'")) | + .metadata.namespace + "/" + .roleRef.name' | sort) + # The leader election binding has no namespace in the template. + got=$(echo "$got" | grep -v '^/') + [ "$got" = "$roles" ] || fail "controller RoleBindings [$got], want [$roles]" + + got=$(echo "$manifests" | "$YQ" -N 'select(.kind == "ClusterRole") | + select(.rules | any_c(.resources | any_c(. == "secrets"))) | .metadata.name') + [ -z "$got" ] || fail "ClusterRoles grant secrets access: [$got]" +} + +"$HELM" dependency build "$CHART_DIR" >/dev/null + +check "$(render)" "$NS" "$NS" +# Duplicates and the release namespace itself must not produce extra RBAC. +check "$(render --set "pullSecretNamespaces={team-a,$NS,team-b,team-a}")" \ + "$NS,team-a,team-b" "$NS" team-a team-b + +for bad in "pullSecretNamespaces=team-a" "pullSecretNamespaces={team-a,}" \ + "pullSecretNamespaces={Team-A}"; do + if render --set "$bad" >/dev/null 2>&1; then fail "accepted --set $bad"; fi +done +echo "ok" diff --git a/internal/controller/bootcnodepool_controller.go b/internal/controller/bootcnodepool_controller.go index 6f9dee32..064fe24b 100644 --- a/internal/controller/bootcnodepool_controller.go +++ b/internal/controller/bootcnodepool_controller.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + "github.com/bootc-dev/bootc-operator/internal/pullsecret" ) // drainStatus tracks an in-progress drain goroutine for a single node. @@ -67,6 +68,11 @@ type BootcNodePoolReconciler struct { TagResolver TagResolver TagResolutionInterval time.Duration + // PullSecretNamespaces is the allowlist of namespaces pools may + // reference pull secrets from. The manager's Secret cache must be + // restricted to the same namespaces. + PullSecretNamespaces []string + // drainCh receives events from drain goroutines to re-enqueue the // owning pool after a drain completes. drainCh chan event.GenericEvent @@ -85,7 +91,7 @@ type BootcNodePoolReconciler struct { // +kubebuilder:rbac:groups=node.bootc.dev,resources=bootcnodes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=node.bootc.dev,resources=bootcnodes/status,verbs=get // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch +// +kubebuilder:rbac:groups="",namespace=system,resources=secrets,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list // +kubebuilder:rbac:groups="",resources=pods/eviction,verbs=create // +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get @@ -701,7 +707,9 @@ func (r *BootcNodePoolReconciler) listAllBootcNodes( } // fetchPullSecretData fetches the .dockerconfigjson data from the pull -// secret referenced by the pool. Returns nil if no pull secret is set. +// secret referenced by the pool. Returns nil if no pull secret is set, and +// an error if the secret's namespace is not in PullSecretNamespaces or the +// reference cannot be stored in the aggregated pull secret. func (r *BootcNodePoolReconciler) fetchPullSecretData( ctx context.Context, pool *bootcv1alpha1.BootcNodePool, @@ -710,6 +718,21 @@ func (r *BootcNodePoolReconciler) fetchPullSecretData( return nil, nil } + if !slices.Contains(r.PullSecretNamespaces, pool.Spec.PullSecretRef.Namespace) { + return nil, fmt.Errorf( + "pull secret namespace %q is not allowed (allowed: %s)", + pool.Spec.PullSecretRef.Namespace, + strings.Join(r.PullSecretNamespaces, ", "), + ) + } + + if _, err := pullsecret.Key( + pool.Spec.PullSecretRef.Namespace, + pool.Spec.PullSecretRef.Name, + ); err != nil { + return nil, err + } + key := types.NamespacedName{ Name: pool.Spec.PullSecretRef.Name, Namespace: pool.Spec.PullSecretRef.Namespace, diff --git a/internal/controller/pullsecret.go b/internal/controller/pullsecret.go new file mode 100644 index 00000000..795d82a1 --- /dev/null +++ b/internal/controller/pullsecret.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "fmt" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ParsePullSecretNamespaces parses a comma-separated namespace list and +// prepends operatorNamespace (when non-empty), which is always allowed. +// Empty entries and surrounding whitespace are dropped and duplicates +// removed. Invalid namespace names are rejected, since the Secret informer +// for them could never sync. +func ParsePullSecretNamespaces(value, operatorNamespace string) ([]string, error) { + var namespaces []string + for _, ns := range append([]string{operatorNamespace}, strings.Split(value, ",")...) { + ns = strings.TrimSpace(ns) + if ns == "" || slices.Contains(namespaces, ns) { + continue + } + if errs := validation.IsDNS1123Label(ns); len(errs) > 0 { + return nil, fmt.Errorf("invalid namespace %q: %s", ns, strings.Join(errs, "; ")) + } + namespaces = append(namespaces, ns) + } + return namespaces, nil +} + +// SecretCacheByObject returns manager cache settings that restrict the +// Secret informer to the given namespaces, so the controller never caches +// (or needs RBAC for) Secrets elsewhere in the cluster. +func SecretCacheByObject(namespaces []string) map[client.Object]cache.ByObject { + config := make(map[string]cache.Config, len(namespaces)) + for _, ns := range namespaces { + config[ns] = cache.Config{} + } + return map[client.Object]cache.ByObject{ + &corev1.Secret{}: {Namespaces: config}, + } +} diff --git a/internal/controller/pullsecret_aggregator.go b/internal/controller/pullsecret_aggregator.go new file mode 100644 index 00000000..d4a822fc --- /dev/null +++ b/internal/controller/pullsecret_aggregator.go @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "bytes" + "context" + "fmt" + "maps" + "slices" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + "github.com/bootc-dev/bootc-operator/internal/pullsecret" +) + +// PullSecretAggregator maintains the aggregated pull secret mounted into the +// daemon: one key per source secret referenced by a pool, holding that +// secret's .dockerconfigjson. It is the Secret's only writer and recomputes +// it from scratch on every reconcile. +type PullSecretAggregator struct { + client.Client + + // Namespace is the operator namespace, which holds the aggregated + // Secret. + Namespace string + + // PullSecretNamespaces is the allowlist of namespaces pools may + // reference pull secrets from. + PullSecretNamespaces []string +} + +// +kubebuilder:rbac:groups="",namespace=system,resources=secrets,verbs=get;list;watch;create +// +kubebuilder:rbac:groups="",namespace=system,resources=secrets,resourceNames=bootc-pull-secrets,verbs=update;patch + +// SetupWithManager sets up the aggregator with the Manager. All events map to +// the single aggregated Secret. +func (r *PullSecretAggregator) SetupWithManager(mgr ctrl.Manager) error { + enqueue := handler.EnqueueRequestsFromMapFunc( + func(context.Context, client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: r.secretKey()}} + }, + ) + // Reconcile once at startup, so the Secret exists even when no pool or + // source secret generates an event. + startup := make(chan event.GenericEvent, 1) + startup <- event.GenericEvent{Object: &corev1.Secret{}} + return ctrl.NewControllerManagedBy(mgr). + Named("pullsecretaggregator"). + WatchesRawSource(source.Channel(startup, enqueue)). + Watches(&bootcv1alpha1.BootcNodePool{}, enqueue, + builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Watches(&corev1.Secret{}, enqueue, + builder.WithPredicates(r.secretPredicate())). + Complete(r) +} + +func (r *PullSecretAggregator) secretKey() types.NamespacedName { + return types.NamespacedName{Namespace: r.Namespace, Name: pullsecret.SecretName} +} + +// secretPredicate accepts source secrets (dockerconfigjson) and the +// aggregated Secret itself, so it is repaired if deleted or edited. +func (r *PullSecretAggregator) secretPredicate() predicate.Predicate { + return predicate.NewPredicateFuncs(func(obj client.Object) bool { + secret, ok := obj.(*corev1.Secret) + if !ok { + return false + } + return secret.Type == corev1.SecretTypeDockerConfigJson || + client.ObjectKeyFromObject(secret) == r.secretKey() + }) +} + +// Reconcile creates or updates the aggregated Secret to match the source +// secrets currently referenced by pools. +func (r *PullSecretAggregator) Reconcile( + ctx context.Context, + _ ctrl.Request, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + desired, err := r.desiredData(ctx) + if err != nil { + return ctrl.Result{}, err + } + + var secret corev1.Secret + err = r.Get(ctx, r.secretKey(), &secret) + if apierrors.IsNotFound(err) { + secret = corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pullsecret.SecretName, + Namespace: r.Namespace, + }, + Type: corev1.SecretTypeOpaque, + Data: desired, + } + if err := r.Create(ctx, &secret); err != nil { + return requeueIfStale(ctx, fmt.Errorf("creating aggregated pull secret: %w", err)) + } + log.Info("Created aggregated pull secret", "keys", len(desired)) + return ctrl.Result{}, nil + } + if err != nil { + return ctrl.Result{}, fmt.Errorf("fetching aggregated pull secret: %w", err) + } + + if maps.EqualFunc(secret.Data, desired, bytes.Equal) { + return ctrl.Result{}, nil + } + secret.Data = desired + if err := r.Update(ctx, &secret); err != nil { + return requeueIfStale(ctx, fmt.Errorf("updating aggregated pull secret: %w", err)) + } + log.Info("Updated aggregated pull secret", "keys", len(desired)) + return ctrl.Result{}, nil +} + +// requeueIfStale requeues without reporting an error when a write failed +// because the cache was behind (typically right after the aggregator's own +// write); the retry recomputes the Secret from scratch. +func requeueIfStale(ctx context.Context, err error) (ctrl.Result, error) { + if apierrors.IsConflict(err) || apierrors.IsAlreadyExists(err) { + logf.FromContext(ctx). + V(1). + Info("Aggregated pull secret changed, retrying", "error", err.Error()) + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return ctrl.Result{}, err +} + +// desiredData returns the aggregated Secret data for all pools. References +// the pool reconciler reports as SecretError (namespace not allowed, name +// too long, secret missing or without .dockerconfigjson) are skipped. +func (r *PullSecretAggregator) desiredData(ctx context.Context) (map[string][]byte, error) { + var pools bootcv1alpha1.BootcNodePoolList + if err := r.List(ctx, &pools); err != nil { + return nil, fmt.Errorf("listing BootcNodePools: %w", err) + } + + desired := map[string][]byte{} + for i := range pools.Items { + ref := pools.Items[i].Spec.PullSecretRef + if ref == nil || !slices.Contains(r.PullSecretNamespaces, ref.Namespace) { + continue + } + key, err := pullsecret.Key(ref.Namespace, ref.Name) + if err != nil { + continue + } + var secret corev1.Secret + err = r.Get(ctx, types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name}, &secret) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return nil, fmt.Errorf("fetching pull secret %s/%s: %w", ref.Namespace, ref.Name, err) + } + if data, ok := secret.Data[corev1.DockerConfigJsonKey]; ok { + desired[key] = data + } + } + return desired, nil +} diff --git a/internal/controller/pullsecret_aggregator_test.go b/internal/controller/pullsecret_aggregator_test.go new file mode 100644 index 00000000..471337d2 --- /dev/null +++ b/internal/controller/pullsecret_aggregator_test.go @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "strings" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + "github.com/bootc-dev/bootc-operator/internal/pullsecret" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +// aggregatedData returns the data of the aggregated pull secret. +func aggregatedData(ctx context.Context) (map[string][]byte, error) { + var secret corev1.Secret + err := k8sClient.Get(ctx, client.ObjectKey{ + Namespace: testutil.OperatorNamespaceName, + Name: pullsecret.SecretName, + }, &secret) + return secret.Data, err +} + +// newAggregatorPool returns a pool that references the given pull secret and +// selects no node, so only the aggregator acts on it. +func newAggregatorPool(name, secretName, secretNamespace string) *bootcv1alpha1.BootcNodePool { + return testutil.NewPool(name, testImageDigestRefA, + testutil.WithNodeSelector(map[string]string{"bootc.dev/aggregator-test": name}), + testutil.WithPullSecret(secretName, secretNamespace), + ) +} + +func TestPullSecretAggregatorAddsKey(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + data := []byte(`{"auths":{"add.example.com":{}}}`) + secret := newDockerConfigSecret("agg-add-secret", testutil.OperatorNamespaceName, data) + g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) + + pool := newAggregatorPool("agg-add-pool", secret.Name, secret.Namespace) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + g.Eventually(aggregatedData).WithArguments(ctx). + Should(HaveKeyWithValue(testutil.OperatorNamespaceName+"_agg-add-secret", data)) +} + +func TestPullSecretAggregatorUpdatesAndRemovesKey(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + key := testutil.OperatorNamespaceName + "_agg-update-secret" + + secret := newDockerConfigSecret("agg-update-secret", testutil.OperatorNamespaceName, + []byte(`{"auths":{"old.example.com":{}}}`)) + g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) + + // Two pools share one source secret, and so one key. + pool1 := newAggregatorPool("agg-update-pool1", secret.Name, secret.Namespace) + g.Expect(k8sClient.Create(ctx, pool1)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool1) }) + pool2 := newAggregatorPool("agg-update-pool2", secret.Name, secret.Namespace) + g.Expect(k8sClient.Create(ctx, pool2)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool2) }) + + g.Eventually(aggregatedData).WithArguments(ctx).Should(HaveKey(key)) + + // Rotating the source secret updates the key. + rotated := []byte(`{"auths":{"new.example.com":{}}}`) + secret.Data[corev1.DockerConfigJsonKey] = rotated + g.Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + g.Eventually(aggregatedData).WithArguments(ctx).Should(HaveKeyWithValue(key, rotated)) + + // The key stays while any pool references the secret. + g.Expect(k8sClient.Delete(ctx, pool1)).To(Succeed()) + g.Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKeyFromObject(pool1), &bootcv1alpha1.BootcNodePool{}) + }).Should(Satisfy(apierrors.IsNotFound)) + g.Consistently(aggregatedData, "2s", pollInterval).WithArguments(ctx).Should(HaveKey(key)) + + // Dropping the last reference removes the key; the Secret is kept. + g.Eventually(func() error { + var p bootcv1alpha1.BootcNodePool + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool2), &p); err != nil { + return err + } + p.Spec.PullSecretRef = nil + return k8sClient.Update(ctx, &p) + }).Should(Succeed()) + g.Eventually(aggregatedData).WithArguments(ctx).ShouldNot(HaveKey(key)) +} + +func TestPullSecretAggregatorSkipsInvalidReferences(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + longName := strings.Repeat("a", 240) + long := newDockerConfigSecret(longName, testutil.OperatorNamespaceName, []byte(`{}`)) + g.Expect(k8sClient.Create(ctx, long)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, long) }) + + noData := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agg-nodata-secret", + Namespace: testutil.OperatorNamespaceName, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"other": []byte("x")}, + } + g.Expect(k8sClient.Create(ctx, noData)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, noData) }) + + valid := newDockerConfigSecret("agg-valid-secret", testutil.OperatorNamespaceName, []byte(`{}`)) + g.Expect(k8sClient.Create(ctx, valid)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, valid) }) + + for _, p := range []*bootcv1alpha1.BootcNodePool{ + // The namespace is not allowed, hence not cached: reading from it + // would fail the reconcile and block the valid key below. + newAggregatorPool("agg-skip-denied", "agg-denied-secret", "agg-denied-ns"), + newAggregatorPool("agg-skip-long", long.Name, long.Namespace), + newAggregatorPool("agg-skip-nodata", noData.Name, noData.Namespace), + newAggregatorPool("agg-skip-missing", "agg-missing-secret", testutil.OperatorNamespaceName), + newAggregatorPool("agg-skip-valid", valid.Name, valid.Namespace), + } { + g.Expect(k8sClient.Create(ctx, p)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, p) }) + } + + // Invalid references are skipped without blocking the valid one. + g.Eventually(aggregatedData).WithArguments(ctx). + Should(HaveKey(testutil.OperatorNamespaceName + "_agg-valid-secret")) + g.Expect(aggregatedData(ctx)).To(And( + Not(HaveKey("agg-denied-ns_agg-denied-secret")), + Not(HaveKey(testutil.OperatorNamespaceName+"_"+longName)), + Not(HaveKey(testutil.OperatorNamespaceName+"_agg-nodata-secret")), + Not(HaveKey(testutil.OperatorNamespaceName+"_agg-missing-secret")), + )) +} + +func TestPullSecretAggregatorRepairsSecret(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + key := testutil.OperatorNamespaceName + "_agg-repair-secret" + + data := []byte(`{"auths":{"repair.example.com":{}}}`) + secret := newDockerConfigSecret("agg-repair-secret", testutil.OperatorNamespaceName, data) + g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) + + pool := newAggregatorPool("agg-repair-pool", secret.Name, secret.Namespace) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + g.Eventually(aggregatedData).WithArguments(ctx).Should(HaveKeyWithValue(key, data)) + + // A manual edit is reverted. + var aggregated corev1.Secret + g.Expect(k8sClient.Get(ctx, client.ObjectKey{ + Namespace: testutil.OperatorNamespaceName, Name: pullsecret.SecretName, + }, &aggregated)).To(Succeed()) + aggregated.Data[key] = []byte("tampered") + aggregated.Data["stray"] = []byte("x") + g.Expect(k8sClient.Update(ctx, &aggregated)).To(Succeed()) + g.Eventually(aggregatedData).WithArguments(ctx).Should(And( + HaveKeyWithValue(key, data), + Not(HaveKey("stray")), + )) + + // A deleted Secret is recreated. + g.Expect(k8sClient.Delete(ctx, &aggregated)).To(Succeed()) + g.Eventually(aggregatedData).WithArguments(ctx).Should(HaveKeyWithValue(key, data)) +} diff --git a/internal/controller/pullsecret_test.go b/internal/controller/pullsecret_test.go index 27b6e1f3..70c4764d 100644 --- a/internal/controller/pullsecret_test.go +++ b/internal/controller/pullsecret_test.go @@ -4,6 +4,7 @@ package controller import ( "context" + "strings" "testing" . "github.com/onsi/gomega" @@ -36,7 +37,7 @@ func TestPullSecretRefPropagation(t *testing.T) { secretData := []byte(`{"auths":{"registry.example.com":{}}}`) - secret := newDockerConfigSecret("ps-prop-secret", "default", secretData) + secret := newDockerConfigSecret("ps-prop-secret", testutil.OperatorNamespaceName, secretData) g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) @@ -46,7 +47,7 @@ func TestPullSecretRefPropagation(t *testing.T) { pool := testutil.NewPool("ps-prop-pool", testImageDigestRefA, testutil.WithWorkerSelector(), - testutil.WithPullSecret("ps-prop-secret", "default"), + testutil.WithPullSecret("ps-prop-secret", testutil.OperatorNamespaceName), ) g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) @@ -56,7 +57,7 @@ func TestPullSecretRefPropagation(t *testing.T) { err := k8sClient.Get(ctx, client.ObjectKey{Name: "ps-prop-node"}, &bn) return bn.Spec.PullSecretRef, err }).Should(Equal(&bootcv1alpha1.PullSecretRef{ - Name: "ps-prop-secret", Namespace: "default", + Name: "ps-prop-secret", Namespace: testutil.OperatorNamespaceName, })) } @@ -72,7 +73,7 @@ func TestPullSecretMissingDegradedAndRecovery(t *testing.T) { pool := testutil.NewPool("ps-missing-pool", testImageDigestRefA, testutil.WithWorkerSelector(), - testutil.WithPullSecret("ps-missing-secret", "default"), + testutil.WithPullSecret("ps-missing-secret", testutil.OperatorNamespaceName), ) g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) @@ -99,7 +100,7 @@ func TestPullSecretMissingDegradedAndRecovery(t *testing.T) { // Create the missing secret and verify recovery. secretData := []byte(`{"auths":{"recovered":{}}}`) - secret := newDockerConfigSecret("ps-missing-secret", "default", secretData) + secret := newDockerConfigSecret("ps-missing-secret", testutil.OperatorNamespaceName, secretData) g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) @@ -121,7 +122,7 @@ func TestPullSecretRemovalClearsRef(t *testing.T) { ctx := context.Background() secretData := []byte(`{"auths":{"remove":{}}}`) - secret := newDockerConfigSecret("ps-remove-secret", "default", secretData) + secret := newDockerConfigSecret("ps-remove-secret", testutil.OperatorNamespaceName, secretData) g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) @@ -131,7 +132,7 @@ func TestPullSecretRemovalClearsRef(t *testing.T) { pool := testutil.NewPool("ps-remove-pool", testImageDigestRefA, testutil.WithWorkerSelector(), - testutil.WithPullSecret("ps-remove-secret", "default"), + testutil.WithPullSecret("ps-remove-secret", testutil.OperatorNamespaceName), ) g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) @@ -156,3 +157,148 @@ func TestPullSecretRemovalClearsRef(t *testing.T) { return bn.Spec.PullSecretRef, err }).Should(BeNil()) } + +func TestPullSecretNamespaceDeniedAndRecovery(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + deniedNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ps-denied-ns"}} + g.Expect(k8sClient.Create(ctx, deniedNS)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, deniedNS) }) + + secretData := []byte(`{"auths":{"denied":{}}}`) + deniedSecret := newDockerConfigSecret("ps-denied-secret", deniedNS.Name, secretData) + g.Expect(k8sClient.Create(ctx, deniedSecret)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, deniedSecret) }) + + node := testutil.NewK8sNode("ps-denied-node", testutil.WorkerLabels()) + g.Expect(k8sClient.Create(ctx, node)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, node) }) + + pool := testutil.NewPool("ps-denied-pool", testImageDigestRefA, + testutil.WithWorkerSelector(), + testutil.WithPullSecret(deniedSecret.Name, deniedSecret.Namespace), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + // The secret exists, but its namespace is not in the allowlist. + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolSecretError), + HaveField("Message", ContainSubstring(`"ps-denied-ns"`)), + ))) + + g.Consistently(func() error { + return k8sClient.Get( + ctx, + client.ObjectKey{Name: "ps-denied-node"}, + &bootcv1alpha1.BootcNode{}, + ) + }, "2s", pollInterval).ShouldNot(Succeed()) + + // Move the secret to the operator namespace and point the pool at it. + allowedSecret := newDockerConfigSecret( + "ps-denied-secret", + testutil.OperatorNamespaceName, + secretData, + ) + g.Expect(k8sClient.Create(ctx, allowedSecret)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, allowedSecret) }) + + g.Eventually(func() error { + var p bootcv1alpha1.BootcNodePool + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &p); err != nil { + return err + } + p.Spec.PullSecretRef.Namespace = testutil.OperatorNamespaceName + return k8sClient.Update(ctx, &p) + }).Should(Succeed()) + + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionFalse), + HaveField("Reason", bootcv1alpha1.PoolHealthy), + ))) + + g.Eventually(func() (*bootcv1alpha1.PullSecretRef, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: "ps-denied-node"}, &bn) + return bn.Spec.PullSecretRef, err + }).Should(Equal(&bootcv1alpha1.PullSecretRef{ + Name: "ps-denied-secret", Namespace: testutil.OperatorNamespaceName, + })) +} + +func TestPullSecretNameTooLong(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + // A valid Secret name whose aggregated key "_" exceeds + // the 253-character Secret key limit. + secretName := strings.Repeat("a", 240) + secret := newDockerConfigSecret(secretName, testutil.OperatorNamespaceName, + []byte(`{"auths":{"long":{}}}`)) + g.Expect(k8sClient.Create(ctx, secret)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, secret) }) + + pool := testutil.NewPool("ps-too-long-pool", testImageDigestRefA, + testutil.WithWorkerSelector(), + testutil.WithPullSecret(secretName, testutil.OperatorNamespaceName), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolSecretError), + HaveField("Message", ContainSubstring("name too long")), + ))) +} + +func TestParsePullSecretNamespaces(t *testing.T) { + tests := []struct { + name string + value string + opNS string + want []string + wantErr bool + }{ + {"empty", "", "op", []string{"op"}, false}, + {"only separators", " , ,", "op", []string{"op"}, false}, + {"unknown operator namespace", "", "", nil, false}, + {"operator namespace prepended", "team-a", "op", []string{"op", "team-a"}, false}, + {"trims and dedupes", " team-a ,,op,team-a", "op", []string{"op", "team-a"}, false}, + {"invalid name", "Team-A", "op", nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + got, err := ParsePullSecretNamespaces(tt.value, tt.opNS) + if tt.wantErr { + g.Expect(err).To(MatchError(ContainSubstring(`invalid namespace "Team-A"`))) + return + } + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index a85cb868..f98878d0 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -60,11 +61,15 @@ func TestMain(m *testing.M) { // Start a manager with the reconciler so controller logic runs // against the envtest API server. + pullSecretNamespaces := []string{testutil.OperatorNamespaceName} mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, Metrics: metricsserver.Options{ BindAddress: "0", // disable metrics server in tests }, + Cache: cache.Options{ + ByObject: SecretCacheByObject(pullSecretNamespaces), + }, }) if err != nil { fmt.Fprintf(os.Stderr, "Failed to create manager: %v\n", err) @@ -84,16 +89,26 @@ func TestMain(m *testing.M) { } testReconciler = &BootcNodePoolReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - KubeClient: kubeClient, - EventNamespace: testutil.OperatorNamespaceName, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + KubeClient: kubeClient, + EventNamespace: testutil.OperatorNamespaceName, + PullSecretNamespaces: pullSecretNamespaces, } if err := testReconciler.SetupWithManager(mgr); err != nil { fmt.Fprintf(os.Stderr, "Failed to setup reconciler: %v\n", err) os.Exit(1) } + if err := (&PullSecretAggregator{ + Client: mgr.GetClient(), + Namespace: testutil.OperatorNamespaceName, + PullSecretNamespaces: pullSecretNamespaces, + }).SetupWithManager(mgr); err != nil { + fmt.Fprintf(os.Stderr, "Failed to setup pull secret aggregator: %v\n", err) + os.Exit(1) + } + mgrCtx, mgrCancel := context.WithCancel(context.Background()) mgrDone := make(chan struct{}) diff --git a/internal/daemon/pullsecret_test.go b/internal/daemon/pullsecret_test.go new file mode 100644 index 00000000..43bb25fc --- /dev/null +++ b/internal/daemon/pullsecret_test.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Apache-2.0 + +package daemon + +import ( + "context" + "errors" + "os" + "path/filepath" + "slices" + "sync" + "testing" + + . "github.com/onsi/gomega" + + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +// authAtStage records the content of /run/ostree/auth.json under the test +// host root each time the fake executor stages, or "" if missing. +type authAtStage struct { + mu sync.Mutex + seen []string +} + +func recordAuthAtStage(fake *fakeExecutor) *authAtStage { + a := &authAtStage{} + fake.setStageHook(func() { + content := "" + if data, err := os.ReadFile(testAuthFile()); err == nil { + content = string(data) + } + a.mu.Lock() + defer a.mu.Unlock() + a.seen = append(a.seen, content) + }) + return a +} + +func (a *authAtStage) get() []string { + a.mu.Lock() + defer a.mu.Unlock() + return slices.Clone(a.seen) +} + +func testAuthFile() string { + return filepath.Join(testHostRoot, "run", "ostree", "auth.json") +} + +func writeMountedPullSecret(t *testing.T, name, namespace, content string) { + t.Helper() + path := filepath.Join(testPullSecretDir, namespace+"_"+name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestStageUsesMountedPullSecret(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + fake := newTestEnv(t) + fake.status = newBootcStatus(testutil.DigestA) + auth := recordAuthAtStage(fake) + + dockerConfig := `{"auths":{"registry.example.com":{}}}` + writeMountedPullSecret(t, "my-secret", "my-ns", dockerConfig) + + bn := testutil.NewNode(testNodeName, testutil.ImageDigestRefB, + testutil.WithNodePullSecret("my-secret", "my-ns")) + g.Expect(k8sClient.Create(ctx, bn)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, bn) }) + + g.Eventually(auth.get).Should(Equal([]string{dockerConfig})) +} + +func writeStaleAuthFile(t *testing.T) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(testAuthFile()), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(testAuthFile(), []byte("stale"), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestStageRemovesAuthFileWithoutMountedEntry(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + fake := newTestEnv(t) + fake.status = newBootcStatus(testutil.DigestA) + auth := recordAuthAtStage(fake) + writeStaleAuthFile(t) + + // The kubelet has not synced the entry yet, or it was never written. + bn := testutil.NewNode(testNodeName, testutil.ImageDigestRefB, + testutil.WithNodePullSecret("not-mounted", "my-ns")) + g.Expect(k8sClient.Create(ctx, bn)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, bn) }) + + g.Eventually(auth.get).Should(Equal([]string{""})) +} + +func TestStageRemovesAuthFileWithoutPullSecretRef(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + fake := newTestEnv(t) + fake.status = newBootcStatus(testutil.DigestA) + auth := recordAuthAtStage(fake) + writeStaleAuthFile(t) + + bn := testutil.NewNode(testNodeName, testutil.ImageDigestRefB) + g.Expect(k8sClient.Create(ctx, bn)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, bn) }) + + g.Eventually(auth.get).Should(Equal([]string{""})) +} + +func TestStageRetryUsesRotatedPullSecret(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + fake := newTestEnv(t) + fake.status = newBootcStatus(testutil.DigestA) + fake.setStageErr(errors.New("unauthorized")) + auth := recordAuthAtStage(fake) + + writeMountedPullSecret(t, "my-secret", "my-ns", "old") + + bn := testutil.NewNode(testNodeName, testutil.ImageDigestRefB, + testutil.WithNodePullSecret("my-secret", "my-ns")) + g.Expect(k8sClient.Create(ctx, bn)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, bn) }) + + g.Eventually(auth.get).Should(Equal([]string{"old"})) + + // The kubelet syncs the rotated secret; the next attempt uses it. + writeMountedPullSecret(t, "my-secret", "my-ns", "new") + fake.setStageErr(nil) + g.Eventually(auth.get).Should(Equal([]string{"old", "new"})) +} diff --git a/internal/daemon/reconciler.go b/internal/daemon/reconciler.go index 7a873336..e0338d1a 100644 --- a/internal/daemon/reconciler.go +++ b/internal/daemon/reconciler.go @@ -6,7 +6,9 @@ package daemon import ( "context" + "errors" "fmt" + "io/fs" "os" "path/filepath" "reflect" @@ -16,12 +18,10 @@ import ( "github.com/distribution/reference" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -34,6 +34,7 @@ import ( bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" "github.com/bootc-dev/bootc-operator/internal/bootc" "github.com/bootc-dev/bootc-operator/internal/image" + "github.com/bootc-dev/bootc-operator/internal/pullsecret" ) const ( @@ -60,12 +61,13 @@ type stageOp struct { // mismatches, and drives updates via bootc stage. type BootcNodeReconciler struct { client.Client - Scheme *runtime.Scheme - NodeName string - HostRoot string + Scheme *runtime.Scheme + NodeName string + HostRoot string + // PullSecretDir is where the aggregated pull secret is mounted. + PullSecretDir string Executor bootc.Executor StatusWatcher *StatusWatcher - APIReader client.Reader inflight stageOp stageDone chan event.GenericEvent @@ -222,7 +224,7 @@ func (r *BootcNodeReconciler) reconcileBootcNode( }, nil } - r.syncPullSecret(ctx, log, bn) + r.syncPullSecret(log, bn) desiredImage := desiredRef.String() @@ -409,47 +411,75 @@ func (r *BootcNodeReconciler) classifyAction( return actionReboot } -// syncPullSecret fetches the pull secret referenced by the BootcNode -// and writes it to the host filesystem so bootc can authenticate. +// syncPullSecret makes the host's auth file mirror the mounted pull secret +// entry for the BootcNode's pullSecretRef, so bootc authenticates with it. +// Without a reference or a readable mounted entry, the auth file is removed +// and bootc falls back to credentials in /etc or /usr/lib. A missing entry is +// not an error: the kubelet may not have synced it yet, and a failed stage is +// retried with backoff. func (r *BootcNodeReconciler) syncPullSecret( - ctx context.Context, log logr.Logger, bn *bootcv1alpha1.BootcNode, ) { - if bn.Spec.PullSecretRef == nil { - return - } - - key := types.NamespacedName{ - Name: bn.Spec.PullSecretRef.Name, - Namespace: bn.Spec.PullSecretRef.Namespace, - } - var secret corev1.Secret - if err := r.APIReader.Get(ctx, key, &secret); err != nil { - log.Error(err, "Failed to fetch pull secret, continuing", "secret", key) - return + data, err := r.mountedPullSecret(bn.Spec.PullSecretRef) + if err != nil { + log.Error(err, "Failed to read mounted pull secret, staging without it") } - data, ok := secret.Data[corev1.DockerConfigJsonKey] - if !ok { - log.Info("Pull secret missing .dockerconfigjson key, continuing", "secret", key) + if data == nil { + if err := r.removeAuthFile(); err != nil { + log.Error(err, "Failed to remove auth file, continuing") + } return } - if err := r.writeAuthFile(data); err != nil { log.Error(err, "Failed to write auth file, continuing") } } -// writeAuthFile writes dockerconfigjson data to the host filesystem -// atomically (temp file + rename). -func (r *BootcNodeReconciler) writeAuthFile(data []byte) error { +// mountedPullSecret returns the mounted pull secret entry for ref, or nil if +// ref is nil or the entry does not exist. +func (r *BootcNodeReconciler) mountedPullSecret(ref *bootcv1alpha1.PullSecretRef) ([]byte, error) { + if ref == nil { + return nil, nil + } + key, err := pullsecret.Key(ref.Namespace, ref.Name) + if err != nil { + return nil, err + } + // Key validates the names, so key cannot contain "/" or be "..". + data, err := os.ReadFile(filepath.Join(r.PullSecretDir, key)) //nolint:gosec // G304 + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + return data, nil +} + +// authDir returns the host directory holding bootc's runtime auth file. +func (r *BootcNodeReconciler) authDir() string { hostRoot := r.HostRoot if hostRoot == "" { hostRoot = "/proc/1/root" } + return filepath.Join(hostRoot, "run", "ostree") +} - dir := filepath.Join(hostRoot, "run", "ostree") +// removeAuthFile removes the host's auth file if present. +func (r *BootcNodeReconciler) removeAuthFile() error { + err := os.Remove(filepath.Join(r.authDir(), "auth.json")) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} + +// writeAuthFile writes dockerconfigjson data to the host filesystem +// atomically (temp file + rename). +func (r *BootcNodeReconciler) writeAuthFile(data []byte) error { + dir := r.authDir() // Standard mode for a host system directory under /run; it only // applies if the host has not created the directory already. if err := os.MkdirAll(dir, 0755); err != nil { //nolint:gosec // G301 diff --git a/internal/daemon/reconciler_test.go b/internal/daemon/reconciler_test.go index 3ea02aec..55932e32 100644 --- a/internal/daemon/reconciler_test.go +++ b/internal/daemon/reconciler_test.go @@ -32,7 +32,7 @@ func TestReconcilePopulatesStatus(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) v1 := "v1" v2 := "v2" @@ -113,7 +113,7 @@ func TestReconcileBootcStatusError(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) fake.setStatusErr(errors.New(bootcStatusErrMsg)) bn := testutil.NewNode(testNodeName, testutil.ImageDigestRefA) @@ -145,7 +145,7 @@ func TestStagingTriggered(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) fake.status = newBootcStatus(testutil.DigestA) bn := testutil.NewNode(testNodeName, testutil.ImageDigestRefB) @@ -186,7 +186,7 @@ func TestStagingError(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) fake.status = newBootcStatus(testutil.DigestA) fake.setStageErr(errors.New(stageErrMsg)) @@ -221,7 +221,7 @@ func TestAlreadyStaged(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) fake.status = newBootcStatus(testutil.DigestA) fake.status.Status.Staged = newBootEntry(testutil.ImageDigestRefB, testutil.DigestB) @@ -250,7 +250,7 @@ func TestRebootingSet(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) fake.status = newBootcStatus(testutil.DigestA) fake.status.Status.Staged = newBootEntry(testutil.ImageDigestRefB, testutil.DigestB) @@ -283,7 +283,7 @@ func TestSoftReboot(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) fake.status = newBootcStatus(testutil.DigestA) fake.status.Status.Staged = newBootEntry(testutil.ImageDigestRefB, testutil.DigestB) @@ -319,7 +319,7 @@ func TestRebootOnlyPolicy(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) fake.status = newBootcStatus(testutil.DigestA) fake.status.Status.Staged = newBootEntry(testutil.ImageDigestRefB, testutil.DigestB) @@ -354,7 +354,7 @@ func TestRollback(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) fake.status = newBootcStatus(testutil.DigestA) fake.status.Status.Staged = newBootEntry(testutil.ImageDigestRefB, testutil.DigestB) @@ -389,7 +389,7 @@ func TestCancelInflightStage(t *testing.T) { g.SetDefaultEventuallyPollingInterval(pollInterval) ctx := context.Background() - fake := newTestEnv() + fake := newTestEnv(t) fake.status = newBootcStatus(testutil.DigestA) firstBlock := make(chan struct{}) diff --git a/internal/daemon/suite_test.go b/internal/daemon/suite_test.go index a8393c93..b249c44b 100644 --- a/internal/daemon/suite_test.go +++ b/internal/daemon/suite_test.go @@ -26,6 +26,11 @@ var ( testEnv *envtest.Environment k8sClient client.Client testReconciler *BootcNodeReconciler + + // testHostRoot and testPullSecretDir stand in for the host filesystem + // and the pull secret mount, so tests never touch the real host. + testHostRoot string + testPullSecretDir string ) func TestMain(m *testing.M) { @@ -64,12 +69,22 @@ func TestMain(m *testing.M) { os.Exit(1) } + tmpDir, err := os.MkdirTemp("", "bootc-daemon-test-") + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create temp dir: %v\n", err) + os.Exit(1) + } + testHostRoot = filepath.Join(tmpDir, "host") + testPullSecretDir = filepath.Join(tmpDir, "pull-secrets") + fake := &fakeExecutor{} testReconciler = &BootcNodeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - NodeName: testNodeName, - Executor: fake, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + NodeName: testNodeName, + HostRoot: testHostRoot, + PullSecretDir: testPullSecretDir, + Executor: fake, StatusWatcher: &StatusWatcher{ Events: make(chan event.GenericEvent, 1), NodeName: testNodeName, @@ -104,13 +119,27 @@ func TestMain(m *testing.M) { if err := testEnv.Stop(); err != nil { fmt.Fprintf(os.Stderr, "Failed to stop envtest: %v\n", err) } + if err := os.RemoveAll(tmpDir); err != nil { + fmt.Fprintf(os.Stderr, "Failed to remove temp dir: %v\n", err) + } os.Exit(code) } // newTestEnv creates a fresh fakeExecutor and StatusWatcher for a test, -// wiring them into the shared reconciler. Each test gets its own cache. -func newTestEnv() *fakeExecutor { +// wiring them into the shared reconciler. Each test gets its own cache and +// an empty host root and pull secret mount. +func newTestEnv(t *testing.T) *fakeExecutor { + t.Helper() + for _, dir := range []string{testHostRoot, testPullSecretDir} { + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + fake := &fakeExecutor{} testReconciler.Executor = fake testReconciler.rebootIssued = false diff --git a/internal/pullsecret/pullsecret.go b/internal/pullsecret/pullsecret.go new file mode 100644 index 00000000..a11af102 --- /dev/null +++ b/internal/pullsecret/pullsecret.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package pullsecret defines the aggregated pull secret shared by the +// controller, which writes it, and the daemon, which reads it from a +// volume mount. +package pullsecret + +import ( + "errors" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/util/validation" +) + +const ( + // SecretName is the name of the aggregated Secret in the operator + // namespace. + SecretName = "bootc-pull-secrets" //nolint:gosec // G101: a name, not a credential + + // MountDir is where the daemon mounts the aggregated Secret. + MountDir = "/var/run/bootc-pull-secrets" +) + +// ErrKeyTooLong is returned by Key when the key exceeds the Secret key length +// limit. +var ErrKeyTooLong = errors.New("name too long") + +// Key returns the aggregated Secret key for the source secret +// namespace/name. "_" cannot appear in namespace or Secret names, so the key +// is unambiguous. An error is returned if namespace or name is not a valid +// object name, since the daemon uses the key as a file name, or if the key +// exceeds the Secret key length limit, since a single invalid key makes the +// API server reject the whole aggregated Secret. +func Key(namespace, name string) (string, error) { + if errs := validation.IsDNS1123Label(namespace); len(errs) > 0 { + return "", fmt.Errorf("pull secret %s/%s: invalid namespace: %s", + namespace, name, strings.Join(errs, "; ")) + } + if errs := validation.IsDNS1123Subdomain(name); len(errs) > 0 { + return "", fmt.Errorf("pull secret %s/%s: invalid name: %s", + namespace, name, strings.Join(errs, "; ")) + } + key := namespace + "_" + name + if len(key) > validation.DNS1123SubdomainMaxLength { + return "", fmt.Errorf( + "pull secret %s/%s: %w (namespace and name must not exceed %d characters combined)", + namespace, + name, + ErrKeyTooLong, + validation.DNS1123SubdomainMaxLength-1, + ) + } + return key, nil +} diff --git a/internal/pullsecret/pullsecret_test.go b/internal/pullsecret/pullsecret_test.go new file mode 100644 index 00000000..56d0e3e0 --- /dev/null +++ b/internal/pullsecret/pullsecret_test.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pullsecret + +import ( + "strings" + "testing" + + . "github.com/onsi/gomega" +) + +func TestKey(t *testing.T) { + tests := []struct { + name string + namespace string + secret string + want string + wantErr bool + }{ + { + "namespace and name", + "bootc-operator", + "my-pull-secret", + "bootc-operator_my-pull-secret", + false, + }, + {"dotted name", "team-a", "registry.example.com", "team-a_registry.example.com", false}, + {"at limit", "ns", strings.Repeat("a", 250), "ns_" + strings.Repeat("a", 250), false}, + {"over limit", "ns", strings.Repeat("a", 251), "", true}, + {"long namespace and name", strings.Repeat("n", 63), strings.Repeat("a", 253), "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + got, err := Key(tt.namespace, tt.secret) + if tt.wantErr { + g.Expect(err).To(MatchError(ErrKeyTooLong)) + return + } + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func TestKeyRejectsInvalidNames(t *testing.T) { + for _, tt := range []struct{ namespace, name string }{ + {"", "secret"}, + {"ns", ""}, + {"..", "secret"}, + {"ns", "../../etc/shadow"}, + {"ns", "a/b"}, + {"ns", "a_b"}, + {"Team-A", "secret"}, + } { + t.Run(tt.namespace+"/"+tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := Key(tt.namespace, tt.name) + g.Expect(err).To(MatchError(ContainSubstring("invalid"))) + }) + } +} diff --git a/test/e2e/pullsecret_namespace_test.go b/test/e2e/pullsecret_namespace_test.go new file mode 100644 index 00000000..eae7c901 --- /dev/null +++ b/test/e2e/pullsecret_namespace_test.go @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "slices" + "testing" + "time" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + "github.com/bootc-dev/bootc-operator/test/e2e/e2eutil" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +// TestPullSecretNamespaces verifies that pools can only use pull secrets +// from the namespaces the operator is configured to read from. The +// operator must already allow E2E_PULL_SECRET_NAMESPACE (make deploy-bink +// does). A pool referencing a secret in a namespace the test creates must +// be Degraded without claiming its node; switched to a secret in the +// allowed namespace, its node must update through the authenticated +// registry. +func TestPullSecretNamespaces(t *testing.T) { + e2eutil.Providers(t, "bink", "eks") + t.Parallel() + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + if env.RegistryUser() == "" || env.RegistryPassword() == "" { + t.Skip("E2E_REGISTRY_USER / E2E_REGISTRY_PASSWORD not set") + } + allowedNS := os.Getenv("E2E_PULL_SECRET_NAMESPACE") + if allowedNS == "" { + t.Skip("E2E_PULL_SECRET_NAMESPACE not set") + } + + ctx := context.Background() + if !slices.Contains(controllerPullSecretNamespaces(t, g, ctx, env.Client), allowedNS) { + t.Skipf("operator does not allow pull secrets from %q", allowedNS) + } + + deniedNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: env.TestID() + "-denied"}} + g.Expect(env.Client.Create(ctx, deniedNS)).To(Succeed()) + t.Cleanup(func() { _ = env.Client.Delete(context.Background(), deniedNS) }) + + nodeName := env.AddNode(t) + + deniedSecret := registrySecret(env, env.TestID()+"-pull-secret", deniedNS.Name) + g.Expect(env.Client.Create(ctx, deniedSecret)).To(Succeed()) + allowedSecret := registrySecret(env, env.TestID()+"-pull-secret", allowedNS) + g.Expect(env.Client.Create(ctx, allowedSecret)).To(Succeed()) + t.Cleanup(func() { _ = env.Client.Delete(context.Background(), allowedSecret) }) + + authImageRef := env.AuthImageRef() + pool := env.NewPool("psns", authImageRef, + testutil.WithPullSecret(deniedSecret.Name, deniedSecret.Namespace), + ) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + + // Phase 1: the secret exists, but its namespace is not allowed. + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := env.Client.Get(ctx, client.ObjectKeyFromObject(pool), &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolSecretError), + HaveField("Message", ContainSubstring(fmt.Sprintf("%q", deniedNS.Name))), + ))) + g.Consistently(func() bool { + var bn bootcv1alpha1.BootcNode + err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return apierrors.IsNotFound(err) + }).WithTimeout(10*time.Second).Should(BeTrue(), + "no BootcNode should be created for a pool with a denied secret") + + t.Logf("Pool %q is Degraded for secret in namespace %q", pool.Name, deniedNS.Name) + + // Phase 2: a secret in the allowed namespace is used to pull the image. + modified := pool.DeepCopy() + modified.Spec.PullSecretRef = &bootcv1alpha1.PullSecretRef{ + Name: allowedSecret.Name, Namespace: allowedSecret.Namespace, + } + g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed()) + *pool = *modified + + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + HaveField("Booted", HaveField("Image", Equal(authImageRef))), + ) + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) + + t.Logf("Node %q updated with a pull secret from namespace %q", nodeName, allowedNS) +} + +// registrySecret returns a dockerconfigjson Secret with the credentials of +// the authenticated e2e registry. +func registrySecret(env *e2eutil.Env, name, namespace string) *corev1.Secret { + auth := base64.StdEncoding.EncodeToString( + []byte(env.RegistryUser() + ":" + env.RegistryPassword()), + ) + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: fmt.Appendf(nil, + `{"auths":{%q:{"auth":%q}}}`, env.RegistryHost(), auth), + }, + } +} diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index da7ff675..95f94b60 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -32,9 +32,9 @@ var operatorCRDNames = []string{ } // TestOperatorUpgrade installs the released version of the operator -// via Helm, verifies basic functionality, then upgrades by running -// helm upgrade with the current image and verifies the operator still -// works. +// from its published Helm chart, verifies basic functionality, then +// upgrades by running helm upgrade with the current chart and image and +// verifies the operator still works. func TestOperatorUpgrade(t *testing.T) { e2eutil.Providers(t, "bink", "eks") @@ -42,10 +42,14 @@ func TestOperatorUpgrade(t *testing.T) { if releaseTag == "" { t.Skip("E2E_OPERATOR_RELEASE_TAG not set") } + // Optional: overrides the image the released chart deploys, e.g. + // with a copy in a local registry. releasedImg := os.Getenv("E2E_OPERATOR_RELEASED_IMG") - if releasedImg == "" { - t.Skip("E2E_OPERATOR_RELEASED_IMG not set") + releasedChart := os.Getenv("E2E_OPERATOR_RELEASED_CHART") + if releasedChart == "" { + t.Skip("E2E_OPERATOR_RELEASED_CHART not set") } + releasedChartVersion := strings.TrimPrefix(releaseTag, "v") g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) @@ -63,24 +67,32 @@ func TestOperatorUpgrade(t *testing.T) { g.Expect(env.Client.Get(ctx, operatorDeployKey(), &origDeploy)).To(Succeed()) currentImg := containerImage(t, origDeploy.Spec.Template.Spec.Containers, "manager") extraArgs := controllerExtraArgs(t, origDeploy.Spec.Template.Spec.Containers) + pullSecretNamespaces := controllerPullSecretNamespaces(t, g, ctx, env.Client) t.Logf("Current operator image: %s", currentImg) - t.Logf("Released operator tag: %s, image: %s", releaseTag, releasedImg) + t.Logf("Released operator tag: %s, image: %s, chart: %s", + releaseTag, releasedImg, releasedChart) - // Phase 1: Uninstall the current operator and re-install with - // the released image via Helm. + // Phase 1: Uninstall the current operator and re-install the + // released version from its published chart, the way users + // upgrading from it have it deployed. The current chart may not work + // with the released image. uninstallOperator(t, g, ctx, env) - helmInstallOperator(t, chartPath, releasedImg, extraArgs) - waitForOperatorReady(t, g, ctx, env.Client) - - t.Logf("Released operator %s installed via Helm", releaseTag) + // Restore the current operator even if the released one fails to + // install, so the tests that run after this one have an operator. t.Cleanup(func() { t.Logf("Restoring operator to current version...") - helmUpgradeOperator(t, chartPath, currentImg, extraArgs) + helmUpgradeOperator(t, chartPath, currentImg, extraArgs, pullSecretNamespaces) waitForOperatorReady(t, g, ctx, env.Client) t.Logf("Operator restored") }) + helmInstallOperator(t, releasedChart, releasedChartVersion, + releasedImg, extraArgs) + waitForOperatorReady(t, g, ctx, env.Client) + + t.Logf("Released operator %s installed via Helm", releaseTag) + // Phase 2: Verify the released version works. nodeName := env.AddNode(t) @@ -92,7 +104,7 @@ func TestOperatorUpgrade(t *testing.T) { t.Logf("Node %q is Idle with released operator", nodeName) // Phase 3: Upgrade via helm upgrade to the current version. - helmUpgradeOperator(t, chartPath, currentImg, extraArgs) + helmUpgradeOperator(t, chartPath, currentImg, extraArgs, pullSecretNamespaces) waitForOperatorReady(t, g, ctx, env.Client) t.Logf("Upgraded operator to current version via Helm") @@ -181,12 +193,17 @@ func uninstallOperator( t.Logf("Operator uninstalled") } -// helmInstallOperator installs the operator via helm install. -func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string) { +// helmInstallOperator installs the given version of a published chart +// via helm install. +func helmInstallOperator( + t *testing.T, + chartRef, chartVersion, img string, + extraArgs []string, +) { t.Helper() - helmRun(t, "dependency", "build", chartPath) args := []string{ - "install", helmReleaseName, chartPath, + "install", helmReleaseName, chartRef, + "--version", chartVersion, "--namespace", testutil.OperatorNamespaceName, "--create-namespace", } @@ -194,25 +211,42 @@ func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string helmRun(t, args...) } -// helmUpgradeOperator upgrades the operator via helm upgrade. -func helmUpgradeOperator(t *testing.T, chartPath, img string, extraArgs []string) { +// helmUpgradeOperator upgrades the operator to the local chart via helm +// upgrade. The release namespace is always allowed, so including it in +// pullSecretNamespaces is harmless. +func helmUpgradeOperator( + t *testing.T, + chartPath, img string, + extraArgs, pullSecretNamespaces []string, +) { t.Helper() + helmRun(t, "dependency", "build", chartPath) args := []string{ "upgrade", "--install", helmReleaseName, chartPath, "--namespace", testutil.OperatorNamespaceName, } args = append(args, helmImageArgs(t, img, extraArgs)...) + if len(pullSecretNamespaces) > 0 { + j, err := json.Marshal(pullSecretNamespaces) + if err != nil { + t.Fatalf("marshaling pullSecretNamespaces: %v", err) + } + args = append(args, "--set-json", "pullSecretNamespaces="+string(j)) + } helmRun(t, args...) } // helmImageArgs returns the --set flags for image and controller args. +// An empty img keeps the image of the chart. func helmImageArgs(t *testing.T, img string, extraArgs []string) []string { t.Helper() - repo, tag := splitImage(img) - args := []string{ - "--set-string", "image.repository=" + repo, - "--set-string", "image.tag=" + tag, - "--set-string", "image.pullPolicy=Always", + args := []string{"--set-string", "image.pullPolicy=Always"} + if img != "" { + repo, tag := splitImage(img) + args = append(args, + "--set-string", "image.repository="+repo, + "--set-string", "image.tag="+tag, + ) } if secret := os.Getenv("E2E_IMAGE_PULL_SECRET"); secret != "" { args = append(args, "--set", "imagePullSecrets[0].name="+secret) @@ -288,7 +322,8 @@ func controllerExtraArgs(t *testing.T, containers []corev1.Container) []string { for _, arg := range c.Args { switch { case arg == "--leader-elect", - strings.HasPrefix(arg, "--health-probe-bind-address="): + strings.HasPrefix(arg, "--health-probe-bind-address="), + strings.HasPrefix(arg, "--pull-secret-namespaces="): default: extra = append(extra, arg) } @@ -299,6 +334,31 @@ func controllerExtraArgs(t *testing.T, containers []corev1.Container) []string { return nil } +// controllerPullSecretNamespaces returns the namespaces the running +// controller allows pull secrets from, as given by its +// --pull-secret-namespaces flag. +func controllerPullSecretNamespaces( + t *testing.T, + g Gomega, + ctx context.Context, + c client.Client, +) []string { + t.Helper() + var d appsv1.Deployment + g.Expect(c.Get(ctx, operatorDeployKey(), &d)).To(Succeed()) + for _, cont := range d.Spec.Template.Spec.Containers { + if cont.Name != "manager" { + continue + } + for _, arg := range cont.Args { + if v, ok := strings.CutPrefix(arg, "--pull-secret-namespaces="); ok { + return strings.Split(v, ",") + } + } + } + return nil +} + // waitForOperatorReady polls until the operator deployment has one available and up-to-date replica. func waitForOperatorReady( t *testing.T,