diff --git a/cmd/main.go b/cmd/main.go index 20ebc767c6..60560c9e46 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,6 +42,7 @@ import ( "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" @@ -521,6 +522,7 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ElasticExternal: discovery.UseExternalElastic(bootConfig), UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, + Extensions: enterprise.New(), } // Before we start any controllers, make sure our options are valid. diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 668756cc10..3b8c5e4197 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -49,6 +49,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" "github.com/tigera/operator/pkg/render/common/authentication" @@ -474,7 +475,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re } // Create a component handler to manage the rendered component. - handler := utils.NewComponentHandler(log, r.client, r.scheme, instance) + handler := utils.NewComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + utils.WithExtensions(r.opts.Extensions), + ) // Render the desired objects from the CRD and create or update them. reqLogger.V(3).Info("rendering components") @@ -497,6 +505,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re KubernetesVersion: r.opts.KubernetesVersion, ClusterDomain: r.opts.ClusterDomain, RequiresAggregationServer: !r.opts.UseV3CRDs, + RequiresQueryServer: installationSpec.Variant.IsEnterprise(), QueryServerTLSKeyPairCertificateManagementOnly: queryServerTLSSecretCertificateManagementOnly, } diff --git a/pkg/controller/apiserver/apiserver_controller_test.go b/pkg/controller/apiserver/apiserver_controller_test.go index b70bd58906..11a4f896de 100644 --- a/pkg/controller/apiserver/apiserver_controller_test.go +++ b/pkg/controller/apiserver/apiserver_controller_test.go @@ -170,6 +170,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -229,6 +230,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -282,6 +284,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, ClusterDomain: dns.DefaultClusterDomain, @@ -307,6 +310,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -329,6 +333,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -353,6 +358,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -375,6 +381,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -400,6 +407,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -427,6 +435,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -452,6 +461,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -478,6 +488,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: false, DetectedProvider: operatorv1.ProviderNone, }, @@ -520,6 +531,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -552,6 +564,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -604,6 +617,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -673,6 +687,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -777,6 +792,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -806,6 +822,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, }, @@ -836,6 +853,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, MultiTenant: true, @@ -883,6 +901,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -927,6 +946,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: false, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -955,6 +975,7 @@ var _ = Describe("apiserver controller tests", func() { tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, EnterpriseCRDExists: true, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: false, diff --git a/pkg/controller/apiserver/apiserver_suite_test.go b/pkg/controller/apiserver/apiserver_suite_test.go index ae0a254ba0..2e1863335b 100644 --- a/pkg/controller/apiserver/apiserver_suite_test.go +++ b/pkg/controller/apiserver/apiserver_suite_test.go @@ -24,8 +24,17 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) +// testExtensions is the enterprise extension Set the API server controller tests +// reconcile with, so the componentHandler applies the API server modifier (query +// server, audit logging, Enterprise RBAC). Reconcilers built in these tests put +// it on their options, mirroring how main wires it in production. +var testExtensions *extensions.Set = enterprise.New() + func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true), zap.Level(uzap.NewAtomicLevelAt(uzap.DebugLevel)))) gomega.RegisterFailHandler(ginkgo.Fail) diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index 4b1b7dc8f6..c51bc19238 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -49,6 +49,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/goldmane" @@ -173,9 +174,9 @@ func newReconciler( scheme: schema, provider: p, status: statusMgr, - clusterDomain: opts.ClusterDomain, tierWatchReady: tierWatchReady, clusterInfoWatchReady: clusterInfoWatchReady, + opts: opts, } c.status.Run(opts.ShutdownContext) return c @@ -190,11 +191,11 @@ type ReconcileConnection struct { scheme *runtime.Scheme provider operatorv1.Provider status status.StatusManager - clusterDomain string tierWatchReady *utils.ReadyFlag clusterInfoWatchReady *utils.ReadyFlag resolvedPodProxies []*httpproxy.Config lastAvailabilityTransition metav1.Time + opts options.ControllerOptions } // Reconcile reads that state of the cluster for a ManagementClusterConnection object and makes changes based on the @@ -283,7 +284,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R log.V(2).Info("Loaded ManagementClusterConnection config", "config", managementClusterConnection) - certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -307,7 +308,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R var guardianKeyPair certificatemanagement.KeyPairInterface if !variant.IsEnterprise() { - guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.clusterDomain) + guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.opts.ClusterDomain) guardianCertificateNames = append(guardianCertificateNames, "localhost", "127.0.0.1") guardianKeyPair, err = certificateManager.GetOrCreateKeyPair(r.cli, render.GuardianKeyPairSecret, whisker.WhiskerNamespace, guardianCertificateNames) if err != nil { @@ -443,7 +444,14 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - ch := utils.NewComponentHandler(log, r.cli, r.scheme, managementClusterConnection) + ch := utils.NewComponentHandler( + log, + r.cli, + r.scheme, + managementClusterConnection, + utils.WithRenderContext(extensions.RenderContext{Installation: installationSpec}), + utils.WithExtensions(r.opts.Extensions), + ) guardianCfg := &render.GuardianConfiguration{ URL: managementClusterConnection.Spec.ManagementClusterAddr, PodProxies: r.resolvedPodProxies, diff --git a/pkg/controller/clusterconnection/clusterconnection_suite_test.go b/pkg/controller/clusterconnection/clusterconnection_suite_test.go index 8967498282..2e6b6feb43 100644 --- a/pkg/controller/clusterconnection/clusterconnection_suite_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_suite_test.go @@ -27,6 +27,7 @@ import ( func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter))) gomega.RegisterFailHandler(ginkgo.Fail) + suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() reporterConfig.JUnitReport = "../../../report/ut/clusterconnection_controller_suite.xml" ginkgo.RunSpecs(t, "pkg/controller/Management Cluster Connection Suite", suiteConfig, reporterConfig) diff --git a/pkg/controller/clusterconnection/shim_test.go b/pkg/controller/clusterconnection/shim_test.go index 7446d229ac..bd5993478d 100644 --- a/pkg/controller/clusterconnection/shim_test.go +++ b/pkg/controller/clusterconnection/shim_test.go @@ -25,6 +25,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/enterprise" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -40,6 +41,7 @@ func NewReconcilerWithShims( ) reconcile.Reconciler { opts := options.ControllerOptions{ ShutdownContext: context.Background(), + Extensions: enterprise.New(), } return newReconciler(cli, schema, status, provider, tierWatchReady, clusterInfoWatchReady, opts) diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index 432ba36b53..5745eafde7 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -174,7 +174,7 @@ type ReconcileGatewayAPI struct { status status.StatusManager clusterDomain string multiTenant bool - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler watchEnvoyProxy func(namespacedName operatorv1.NamespacedName) error watchEnvoyGateway func(namespacedName operatorv1.NamespacedName) error watchGateways func() error diff --git a/pkg/controller/gatewayapi/gatewayapi_controller_test.go b/pkg/controller/gatewayapi/gatewayapi_controller_test.go index 391160c534..bd02010623 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller_test.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller_test.go @@ -779,7 +779,7 @@ var _ = Describe("Gateway API controller tests", func() { var fakeComponentHandlers []*fakeComponentHandler -func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler { +func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler { h := &fakeComponentHandler{ client: client, scheme: scheme, diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index c4dcdd6aaa..b01f01cfca 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -43,7 +43,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -64,7 +63,6 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/controller/gatewayapi" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration" @@ -75,29 +73,20 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/resourcequota" "github.com/tigera/operator/pkg/render/goldmane" "github.com/tigera/operator/pkg/render/kubecontrollers" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) const ( techPreviewFeatureSeccompApparmor = "tech-preview.operator.tigera.io/node-apparmor-profile" - - // The default port used by calico/node to report Calico Enterprise internal metrics. - // This is separate from the calico/node prometheus metrics port, which is user configurable. - defaultNodeReporterPort = 9081 - - defaultFelixMetricsDefaultPort = 9091 ) const InstallationName string = "calico" @@ -215,13 +204,6 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } // Watch for changes to KubeControllersConfiguration. - // Watch GatewayAPI: spec.extensions.waf.state gates the WAF v3 surface on - // calico-kube-controllers. See design tigera/designs#25 (PMREQ-384) §Gating. - if err := c.WatchObject(&operatorv1.GatewayAPI{}, &handler.EnqueueRequestForObject{}); err != nil { - log.V(5).Info("Failed to create GatewayAPI watch", "err", err) - return fmt.Errorf("core-controller failed to watch operator GatewayAPI resource: %w", err) - } - err = c.WatchObject(&v3.KubeControllersConfiguration{}, &handler.EnqueueRequestForObject{}) if err != nil { return fmt.Errorf("tigera-installation-controller failed to watch KubeControllersConfiguration resource: %w", err) @@ -240,27 +222,8 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } if opts.EnterpriseCRDExists { - // Watch for changes to primary resource ManagementCluster - err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch for changes to primary resource ManagementClusterConnection - err = c.WatchObject(&operatorv1.ManagementClusterConnection{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // watch for change to primary resource LogCollector - err = c.WatchObject(&operatorv1.LogCollector{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch the internal manager TLS secret in the operator namespace, which included in the bundle for es-kube-controllers. - if err = utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()); err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch secret: %v", err) + if err = opts.Extensions.SetupWatches(extensions.InstallationController, c); err != nil { + return fmt.Errorf("tigera-installation-controller failed to set up extension watches: %w", err) } if opts.ManageCRDs { @@ -336,25 +299,17 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions) (*Reconc typhaScaler := newTyphaAutoscaler(opts.K8sClientset, nodeIndexInformer, typhaListWatch, statusManager) r := &ReconcileInstallation{ - config: mgr.GetConfig(), - client: mgr.GetClient(), - clientset: opts.K8sClientset, - scheme: mgr.GetScheme(), - shutdownContext: opts.ShutdownContext, - watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, - status: statusManager, - typhaAutoscaler: typhaScaler, - namespaceMigration: nm, - enterpriseCRDsExist: opts.EnterpriseCRDExists, - clusterDomain: opts.ClusterDomain, - manageCRDs: opts.ManageCRDs, - tierWatchReady: &utils.ReadyFlag{}, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, - v3CRDs: opts.UseV3CRDs, - kubernetesVersion: opts.KubernetesVersion, - apiDiscovery: opts.APIDiscovery, + config: mgr.GetConfig(), + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + watches: make(map[runtime.Object]struct{}), + status: statusManager, + typhaAutoscaler: typhaScaler, + namespaceMigration: nm, + tierWatchReady: &utils.ReadyFlag{}, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, + opts: opts, } r.status.Run(opts.ShutdownContext) r.typhaAutoscaler.start(opts.ShutdownContext) @@ -396,27 +351,19 @@ type ReconcileInstallation struct { // that reads objects from the cache and writes to the apiserver config *rest.Config client client.Client - clientset *kubernetes.Clientset scheme *runtime.Scheme - shutdownContext context.Context watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager typhaAutoscaler *typhaAutoscaler typhaAutoscalerNonClusterHost *typhaAutoscaler namespaceMigration migration.NamespaceMigration - enterpriseCRDsExist bool migrationChecked bool - clusterDomain string - manageCRDs bool tierWatchReady *utils.ReadyFlag migrationWatchReady *utils.ReadyFlag - v3CRDs bool - kubernetesVersion *common.VersionInfo - apiDiscovery *discovery.APIDiscovery + opts options.ControllerOptions // newComponentHandler returns a new component handler. Useful stub for unit testing. - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler } // GetActivePools returns the full set of enabled IP pools in the cluster. @@ -857,7 +804,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // update Installation with defaults - if err := updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider); err != nil { + if err := updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider); err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying installation", err, reqLogger) return reconcile.Result{}, err } @@ -1021,10 +968,10 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // The operator supports running in a "Calico only" mode so that it doesn't need to run enterprise-specific controllers. // If we are switching from this mode to one that enables enterprise, we need to restart the operator to enable the other controllers. - if !r.enterpriseCRDsExist && instance.Spec.Variant.IsEnterprise() { + if !r.opts.EnterpriseCRDExists && instance.Spec.Variant.IsEnterprise() { // Perform an API discovery to determine if the necessary APIs exist. If they do, we can reboot into enterprise mode. // if they do not, we need to notify the user that the requested configuration is invalid. - b, err := discovery.RequiresTigeraSecure(r.clientset) + b, err := discovery.RequiresTigeraSecure(r.opts.K8sClientset) if b { log.Info("Rebooting to enable TigeraSecure controllers") os.Exit(0) @@ -1049,16 +996,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile var managementCluster *operatorv1.ManagementCluster var managementClusterConnection *operatorv1.ManagementClusterConnection - var logCollector *operatorv1.LogCollector - if r.enterpriseCRDsExist { - logCollector, err = utils.GetLogCollector(ctx, r.client) - if logCollector != nil { - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading LogCollector", err, reqLogger) - return reconcile.Result{}, err - } - } - + if r.opts.EnterpriseCRDExists { managementCluster, err = utils.GetManagementCluster(ctx, r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) @@ -1096,7 +1034,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -1109,18 +1047,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - if instance.Spec.Variant.IsEnterprise() { - managerInternalTLSSecret, err := certificateManager.GetCertificate(r.client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error fetching TLS secret %s in namespace %s", render.ManagerInternalTLSSecretName, common.OperatorNamespace()), err, reqLogger) - return reconcile.Result{}, nil - } else if managerInternalTLSSecret != nil { - // It may seem odd to add the manager internal TLS secret to the trusted bundle for Typha / calico-node, but this bundle is also used - // for other components in this namespace such as es-kube-controllers, who communicates with Voltron and thus needs to trust this certificate. - typhaNodeTLS.TrustedBundle.AddCertificates(managerInternalTLSSecret) - } - } - birdTemplates, err := getBirdTemplates(r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error retrieving confd templates", err, reqLogger) @@ -1221,93 +1147,37 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. - nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface calicoVersion := components.CalicoRelease - - felixPrometheusMetricsPort := defaultFelixMetricsDefaultPort - if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - - if felixConfiguration.Spec.PrometheusMetricsPort != nil { - felixPrometheusMetricsPort = *felixConfiguration.Spec.PrometheusMetricsPort - } - - nodePrometheusTLS, err = certificateManager.GetOrCreateKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - if nodePrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) - } - prometheusClientCert, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, "Unable to fetch prometheus certificate", err, reqLogger) - return reconcile.Result{}, err - } - if prometheusClientCert != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(prometheusClientCert) - } - - // es-kube-controllers needs to trust the ESGW certificate. We'll fetch it here and add it to the trusted bundle. - // Note that although we're adding this to the typhaNodeTLS trusted bundle, it will be used by es-kube-controllers. This is because - // all components within this namespace share a trusted CA bundle. This is necessary because prior to v3.13 secrets were not signed by - // a single CA so we need to include each individually. - esgwCertificate, err := certificateManager.GetCertificate(r.client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve / validate %s", relasticsearch.PublicCertSecret), err, reqLogger) - return reconcile.Result{}, err - } - if esgwCertificate != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(esgwCertificate) - } - calicoVersion = components.EnterpriseRelease } - kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, + Controller: extensions.InstallationController, + Ctx: ctx, + Client: r.client, + CertificateManager: certificateManager, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) + return reconcile.Result{}, err + } + renderCtx, managedKeyPairs, err := r.opts.Extensions.ExtendContext(cc) if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to read KubeControllersConfiguration", err, reqLogger) + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) return reconcile.Result{}, err } - // Secure calico kube controller metrics. - var kubeControllerTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - // Create or Get TLS certificates for kube controller. - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error finding or creating TLS certificate kube controllers metric", err, reqLogger) - return reconcile.Result{}, err - } - - // Add prometheus client certificate to Trusted bundle. - kubeControllerPrometheusTLS, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate for kube controllers", err, reqLogger) - return reconcile.Result{}, err - } else if kubeControllerPrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(kubeControllerTLS, kubeControllerPrometheusTLS) - } + kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to read KubeControllersConfiguration", err, reqLogger) + return reconcile.Result{}, err } nodeAppArmorProfile := "" @@ -1317,7 +1187,14 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance) + handler := r.newComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithRenderContext(renderCtx), + utils.WithExtensions(r.opts.Extensions), + ) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ @@ -1374,49 +1251,14 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } - // Read the GatewayAPI CR (if present) to decide whether to render the WAF - // v3 (Gateway API add-on) surface — env vars, RBAC, applicationlayer - // reconciler, and the in-process admission webhook — on - // calico-kube-controllers. Default-off: if no GatewayAPI CR exists or - // spec.extensions.waf.state != Enabled, the WAF surface is not rendered. - // See design tigera/designs#25 (PMREQ-384) §Gating. - wafGatewayExtensionEnabled := false - if gatewayAPI, msg, err := gatewayapi.GetGatewayAPI(ctx, r.client); err == nil { - wafGatewayExtensionEnabled = gatewayAPI.Spec.IsWAFGatewayExtensionEnabled() - } else if !apierrors.IsNotFound(err) { - // Mirrors the GatewayAPI controller's handling: a read error or a - // duplicate default/tigera-secure pair degrades rather than guessing. - r.status.SetDegraded(operatorv1.ResourceReadError, msg, err, reqLogger) - return reconcile.Result{}, err - } - - // When the WAF v3 surface is enabled, issue the serving cert for the - // in-process WAF admission webhook (hosted by calico-kube-controllers, - // fronted by the tigera-waf-webhook Service). It is materialized into - // calico-system alongside the other kube-controllers certs below and mounted - // into the Pod by the kube-controllers render. - var wafWebhookTLS certificatemanagement.KeyPairInterface - if wafGatewayExtensionEnabled { - wafWebhookTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating WAF admission webhook TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - keyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), - rcertificatemanagement.NewKeyPairOption(nodePrometheusTLS, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), - rcertificatemanagement.NewKeyPairOption(kubeControllerTLS, true, true), - // Nil when the WAF v3 surface is disabled; the certificate-management - // render skips nil key pairs. - rcertificatemanagement.NewKeyPairOption(wafWebhookTLS, true, true), + } + // Manage any key pairs the variant extension created controller-side. + for _, kp := range managedKeyPairs { + keyPairOptions = append(keyPairOptions, rcertificatemanagement.NewKeyPairOption(kp, true, true)) } components = append(components, @@ -1460,11 +1302,11 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile hepListWatch := cache.NewListWatchFromClient(calicoClient.ProjectcalicoV3().RESTClient(), "hostendpoints", corev1.NamespaceAll, fields.Everything()) hepIndexInformer := cache.NewSharedIndexInformer(hepListWatch, &v3.HostEndpoint{}, 0, cache.Indexers{}) - go hepIndexInformer.Run(r.shutdownContext.Done()) + go hepIndexInformer.Run(r.opts.ShutdownContext.Done()) - typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.clientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) - r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.clientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) - r.typhaAutoscalerNonClusterHost.start(r.shutdownContext) + typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.opts.K8sClientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) + r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.opts.K8sClientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) + r.typhaAutoscalerNonClusterHost.start(r.opts.ShutdownContext) } } } @@ -1476,7 +1318,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile Installation: &instance.Spec, TLS: typhaNodeTLS, MigrateNamespaces: needsNamespaceMigration, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, NonClusterHost: nonclusterhost, FelixHealthPort: *felixConfiguration.Spec.HealthPort, } @@ -1592,28 +1434,24 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // Build a configuration for rendering calico/node. nodeCfg := render.NodeConfiguration{ - GoldmaneRunning: goldmaneRunning, - K8sServiceEp: k8sapi.Endpoint, - Installation: &instance.Spec, - IPPools: crdPoolsToOperator(currentPools.Items), - LogCollector: logCollector, - BirdTemplates: birdTemplates, - TLS: typhaNodeTLS, - ClusterDomain: r.clusterDomain, - DefaultDNSPolicy: defaultDNSPolicy, - DefaultDNSConfig: defaultDNSConfig, - GoldmaneIP: goldmaneIP, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - NodeAppArmorProfile: nodeAppArmorProfile, - MigrateNamespaces: needsNamespaceMigration, - CanRemoveCNIFinalizer: canRemoveCNI, - PrometheusServerTLS: nodePrometheusTLS, - FelixHealthPort: *felixConfiguration.Spec.HealthPort, - NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, - FelixPrometheusMetricsEnabled: utils.IsFelixPrometheusMetricsEnabled(felixConfiguration), - FelixPrometheusMetricsPort: felixPrometheusMetricsPort, - V3CRDs: r.v3CRDs, + GoldmaneRunning: goldmaneRunning, + K8sServiceEp: k8sapi.Endpoint, + Installation: &instance.Spec, + IPPools: crdPoolsToOperator(currentPools.Items), + BirdTemplates: birdTemplates, + TLS: typhaNodeTLS, + ClusterDomain: r.opts.ClusterDomain, + DefaultDNSPolicy: defaultDNSPolicy, + DefaultDNSConfig: defaultDNSConfig, + GoldmaneIP: goldmaneIP, + BGPLayouts: bgpLayout, + NodeAppArmorProfile: nodeAppArmorProfile, + MigrateNamespaces: needsNamespaceMigration, + CanRemoveCNIFinalizer: canRemoveCNI, + FelixHealthPort: *felixConfiguration.Spec.HealthPort, + NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, + V3CRDs: r.opts.UseV3CRDs, + ImageOverrides: r.opts.Extensions.Images(), } if bgpConfiguration.Spec.BindMode != nil { @@ -1657,55 +1495,21 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } components = append(components, render.CSI(&csiCfg)) - // Build a configuration for rendering calico/kube-controllers. - // Provision a dedicated WAF wasm pull secret so the WAF reconciler - // replicates it into tenant namespaces without clashing with the - // operator-managed tigera-pull-secret the GatewayAPI render also copies - // there (EV-6386). The EnvoyExtensionPolicy image source takes a single - // pullSecretRef, so the registry auths of all Installation pull secrets - // are merged into it rather than picking one. - var wasmPullSecret *corev1.Secret - if wafGatewayExtensionEnabled && len(pullSecrets) > 0 { - var skipped []string - wasmPullSecret, skipped = kubecontrollers.MergeWAFPullSecret(pullSecrets) - if len(skipped) > 0 { - reqLogger.Info("Skipped unparseable imagePullSecrets when building the WAF wasm pull secret", "skipped", skipped) - } - } - // Provision the dedicated WAF wasm CA-bundle ConfigMap as a renamed copy of - // the trusted CA bundle, so the WAF reconciler replicates it into tenant - // namespaces for the Coraza wasm OCI registry TLS check without clashing with - // the operator-managed tigera-ca-bundle the GatewayAPI render also copies - // there (EV-6386). The dedicated source was previously a TODO; the full - // TrustedBundle (not the RO interface the kube-controllers render sees) is - // available here, so build it in the core controller. - var wasmCACert *corev1.ConfigMap - if wafGatewayExtensionEnabled { - wasmCACert = typhaNodeTLS.TrustedBundle.ConfigMap(common.CalicoNamespace) - wasmCACert.Name = kubecontrollers.WASMCACertName - } + // Build a configuration for rendering calico/kube-controllers. The Calico + // Enterprise surface (extra RBAC, enterprise controllers, metrics TLS, and the + // WAF v3 / Gateway API add-on) is layered on by the enterprise extension. kubeControllersCfg := kubecontrollers.KubeControllersConfiguration{ K8sServiceEp: k8sapi.Endpoint, K8sServiceEpPodNetwork: k8sapi.PodNetworkEndpoint, Installation: &instance.Spec, ManagementCluster: managementCluster, ManagementClusterConnection: managementClusterConnection, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, MetricsPort: kubeControllersMetricsPort, Terminating: installationMarkedForDeletion, - MetricsServerTLS: kubeControllerTLS, TrustedBundle: typhaNodeTLS.TrustedBundle, Namespace: common.CalicoNamespace, BindingNamespaces: []string{common.CalicoNamespace}, - WAFGatewayExtensionEnabled: wafGatewayExtensionEnabled, - WAFWebhookServerTLS: wafWebhookTLS, - WASMPullSecret: wasmPullSecret, - WASMCACert: wasmCACert, - // The webhook Service + ValidatingWebhookConfiguration are rendered by - // the kube-controllers component (and deleted when the WAF extension is - // disabled); the caBundle is the operator CA that issued the serving - // cert above. - WAFWebhookCABundle: certificateManager.KeyPair().GetCertificatePEM(), } components = append(components, kubecontrollers.NewCalicoKubeControllers(&kubeControllersCfg)) @@ -1841,13 +1645,15 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.ReadyToMonitor() // Check BYO certificate expiry warnings and propagate them to the status manager. - certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{ + keyPairWarnings := map[string]certificatemanagement.KeyPairInterface{ render.TyphaTLSSecretName: typhaNodeTLS.TyphaSecret, render.NodeTLSSecretName: typhaNodeTLS.NodeSecret, render.TyphaTLSSecretName + render.TyphaNonClusterHostSuffix: typhaNodeTLS.TyphaSecretNonClusterHost, - render.NodePrometheusTLSServerSecret: nodePrometheusTLS, - kubecontrollers.KubeControllerPrometheusTLSSecret: kubeControllerTLS, - }, r.status) + } + for _, kp := range managedKeyPairs { + keyPairWarnings[kp.GetName()] = kp + } + certificatemanagement.CheckKeyPairWarnings(keyPairWarnings, r.status) // We can clear the degraded state now since as far as we know everything is in order. r.status.ClearDegraded() @@ -2348,10 +2154,10 @@ func (r *ReconcileInstallation) checkActive(log logr.Logger) (*corev1.ConfigMap, } func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operatorv1.ProductVariant, log logr.Logger) error { - if !r.manageCRDs { + if !r.opts.ManageCRDs { return nil } - crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.v3CRDs)...)...) + crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.opts.UseV3CRDs)...)...) // Specify nil for the CR so no ownership is put on the CRDs. We do this so removing the // Installation CR will not remove the CRDs. handler := r.newComponentHandler(log, r.client, r.scheme, nil) @@ -2363,19 +2169,19 @@ func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operator } func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // MutatingAdmissionPolicy served version was discovered once at startup (v1 was promoted to GA // in k8s 1.36 and v1beta1 (introduced in 1.32) is scheduled for removal in 1.37). - mapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) + mapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) if mapAPIVersion == "" { r.status.SetDegraded(operatorv1.ResourceNotReady, "Kubernetes cluster does not serve MutatingAdmissionPolicy (requires v1.32+); policy defaulting will not be available", nil, log) return nil } - desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, mapAPIVersion) + desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, mapAPIVersion) existingMAPs, existingMAPBs, err := admission.ListManaged(ctx, r.client, mapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed MutatingAdmissionPolicy resources", err, log) @@ -2386,20 +2192,20 @@ func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Cont } func (r *ReconcileInstallation) updateValidatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // ValidatingAdmissionPolicy reached GA (v1) well before MutatingAdmissionPolicy, so it has its own // served version and is reconciled independently of whether the cluster serves MAPs. If the cluster // doesn't serve it at all there's nothing to do, so skip rather than degrade. - vapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) + vapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) if vapAPIVersion == "" { log.Info("Kubernetes cluster does not serve ValidatingAdmissionPolicy, skipping") return nil } - desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, vapAPIVersion) + desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, vapAPIVersion) existingVAPs, existingVAPBs, err := admission.ListManagedValidating(ctx, r.client, vapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed ValidatingAdmissionPolicy resources", err, log) diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index bda4f6065b..3fcd559551 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -56,6 +56,7 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -190,18 +191,21 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -819,19 +823,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -1041,18 +1048,21 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2217,7 +2227,7 @@ var _ = Describe("Testing core-controller installation", func() { cr.Spec.Variant = operator.Calico cr.Status.Variant = operator.Calico Expect(c.Create(ctx, cr)).NotTo(HaveOccurred()) - r.enterpriseCRDsExist = false + r.opts.EnterpriseCRDExists = false Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) _, err := r.Reconcile(ctx, reconcile.Request{}) @@ -2329,19 +2339,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2466,18 +2479,21 @@ var _ = Describe("Testing core-controller installation", func() { componentHandler = newFakeComponentHandler() r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - enterpriseCRDsExist: true, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2629,13 +2645,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1 MAPs when v1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2660,13 +2679,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1beta1 MAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2689,13 +2711,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1alpha1 MAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2718,13 +2743,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when no served version exists and should set degraded", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2736,13 +2764,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2753,13 +2784,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when manageCRDs=false", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: false, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: false, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2783,13 +2817,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - client: clientFor(staleMAP, staleMAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleMAP, staleMAPB), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2825,13 +2862,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - client: clientFor(initial...), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(initial...), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2843,13 +2883,16 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2914,13 +2957,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1 VAPs when v1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2945,13 +2991,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1beta1 VAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2974,13 +3023,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1alpha1 VAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2991,13 +3043,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should skip without degrading when no served version exists", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3009,13 +3064,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should not create VAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3039,13 +3097,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - client: clientFor(staleVAP, staleVAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleVAP, staleVAPB), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3063,13 +3124,16 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } diff --git a/pkg/controller/installation/installation_controller_suite_test.go b/pkg/controller/installation/installation_controller_suite_test.go index 4924f3468b..f589361c5a 100644 --- a/pkg/controller/installation/installation_controller_suite_test.go +++ b/pkg/controller/installation/installation_controller_suite_test.go @@ -25,8 +25,17 @@ import ( clientfeaturestesting "k8s.io/client-go/features/testing" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" ) +// testExtensions is the enterprise extension Set the installation controller +// tests reconcile with, mirroring how main wires it in production. Reconcilers +// built in these tests put it on their options so the node image overrides and +// modifiers apply. +var testExtensions *extensions.Set = enterprise.New() + func TestInstallation(t *testing.T) { // Disable WatchListClient for tests. In client-go v0.35+, this feature defaults to true and // causes informers to wait for bookmark events that fake clients never send, leading to timeouts. diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index e23bf55db7..1c05e72227 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -16,7 +16,6 @@ package installation import ( "context" - "errors" "fmt" "reflect" @@ -50,10 +49,8 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/monitor" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var logw = logf.Log.WithName("controller_windows") @@ -82,7 +79,7 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e return fmt.Errorf("tigera-windows-controller failed to watch calico Tigerastatus: %w", err) } - if ri.autoDetectedProvider.IsOpenShift() { + if ri.opts.DetectedProvider.IsOpenShift() { // Watch for openshift network configuration as well. If we're running in OpenShift, we need to // merge this configuration with our own and the write back the status object. err = c.WatchObject(&configv1.Network{}, &handler.EnqueueRequestForObject{}) @@ -150,14 +147,9 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e // Watch for changes to IPAMConfiguration. go utils.WaitToAddResourceWatch(c, opts.K8sClientset, logw, ri.ipamConfigWatchReady, []client.Object{&v3.IPAMConfiguration{TypeMeta: metav1.TypeMeta{Kind: v3.KindIPAMConfiguration}}}) - if ri.enterpriseCRDsExist { - for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { - if err = utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", render.NodePrometheusTLSServerSecret, ns, err) - } - if err = utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", monitor.PrometheusClientTLSSecretName, ns, err) - } + if ri.opts.EnterpriseCRDExists { + if err = ri.opts.Extensions.SetupWatches(extensions.WindowsController, c); err != nil { + return fmt.Errorf("tigera-windows-controller failed to set up extension watches: %w", err) } } @@ -176,11 +168,9 @@ type ReconcileWindows struct { client client.Client scheme *runtime.Scheme watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager - enterpriseCRDsExist bool - clusterDomain string ipamConfigWatchReady *utils.ReadyFlag + opts options.ControllerOptions } // newWindowsReconciler returns a new reconcile.Reconciler @@ -192,11 +182,9 @@ func newWindowsReconciler(mgr manager.Manager, opts options.ControllerOptions) ( client: mgr.GetClient(), scheme: mgr.GetScheme(), watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, status: statusManager, - enterpriseCRDsExist: opts.EnterpriseCRDExists, - clusterDomain: opts.ClusterDomain, ipamConfigWatchReady: &utils.ReadyFlag{}, + opts: opts, } r.status.Run(opts.ShutdownContext) return r, nil @@ -288,7 +276,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace()) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -328,36 +316,9 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. - nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - - // The key pair is created by the core controller, so if it isn't set, requeue to wait until it is - nodePrometheusTLS, err = certificateManager.GetKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error getting TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - var component render.Component - kubeDNSServiceName := utils.GetDNSServiceName(r.autoDetectedProvider) + kubeDNSServiceName := utils.GetDNSServiceName(r.opts.DetectedProvider) kubeDNSService := &corev1.Service{} err = r.client.Get(ctx, kubeDNSServiceName, kubeDNSService) if err != nil { @@ -377,15 +338,38 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } + // Run the variant's windows controller extension to build the render context + // (creating no enterprise artifacts in core). + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, + Controller: extensions.WindowsController, + Ctx: ctx, + Client: r.client, + CertificateManager: certificateManager, + } + if err := r.opts.Extensions.Validate(cc); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid installation configuration", err, reqLogger) + return reconcile.Result{}, err + } + renderCtx, _, err := r.opts.Extensions.ExtendContext(cc) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing windows extension", err, reqLogger) + return reconcile.Result{}, err + } + windowsCfg := render.WindowsConfiguration{ - K8sServiceEp: k8sapi.Endpoint, - K8sDNSServers: kubeDNSIPs, - Installation: &instance.Spec, - ClusterDomain: r.clusterDomain, - TLS: typhaNodeTLS, - PrometheusServerTLS: nodePrometheusTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + K8sServiceEp: k8sapi.Endpoint, + K8sDNSServers: kubeDNSIPs, + Installation: &instance.Spec, + ClusterDomain: r.opts.ClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + ImageOverrides: r.opts.Extensions.Images(), } component = render.Windows(&windowsCfg) @@ -406,7 +390,14 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } // Create a component handler to create or update the rendered components. - handler := utils.NewComponentHandler(logw, r.client, r.scheme, instance) + handler := utils.NewComponentHandler( + logw, + r.client, + r.scheme, + instance, + utils.WithRenderContext(renderCtx), + utils.WithExtensions(r.opts.Extensions), + ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error creating / updating resource", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/controller/installation/windows_controller_test.go b/pkg/controller/installation/windows_controller_test.go index ae5866bfa5..6fa1d2cf0d 100644 --- a/pkg/controller/installation/windows_controller_test.go +++ b/pkg/controller/installation/windows_controller_test.go @@ -29,6 +29,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -119,12 +120,15 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - enterpriseCRDsExist: true, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -155,7 +159,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider)).NotTo(HaveOccurred()) certificateManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) prometheusTLS, err := certificateManager.GetOrCreateKeyPair(c, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace(), []string{monitor.PrometheusClientTLSSecretName}) @@ -194,7 +198,7 @@ var _ = Describe("windows-controller installation tests", func() { cr.Status = operator.InstallationStatus{ Variant: operator.Calico, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider)).NotTo(HaveOccurred()) // Set serviceCIDRs in the installation (required for Calico for Windows) cr.Spec.ServiceCIDRs = []string{"10.96.0.0/12"} @@ -609,12 +613,15 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + EnterpriseCRDExists: true, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - enterpriseCRDsExist: true, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -663,7 +670,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider)).NotTo(HaveOccurred()) Expect(c.Create(ctx, instance)).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/pkg/controller/logstorage/common/common.go b/pkg/controller/logstorage/common/common.go index 2c070439d7..e15f982b49 100644 --- a/pkg/controller/logstorage/common/common.go +++ b/pkg/controller/logstorage/common/common.go @@ -26,7 +26,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/crypto" - "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/enterprise" ) const ( @@ -45,7 +45,7 @@ const ( // the gateway credentials, and a secret containing real admin level credentials is created and stored in the tigera-elasticsearch namespace to be swapped in once // ES Gateway has confirmed that the gateway credentials match. func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1.Secret, esAdminUserName string, cli client.Client, h utils.NamespaceHelper) (*corev1.Secret, *corev1.Secret, *corev1.Secret, error) { - kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) + kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) if err != nil { return nil, nil, nil, err } @@ -53,11 +53,11 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 password := crypto.GeneratePassword(16) kubeControllersGatewaySecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, + Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: h.TruthNamespace(), }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(enterprise.ElasticsearchKubeControllersUserName), "password": []byte(password), }, } @@ -67,34 +67,34 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 return nil, nil, nil, err } - kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) + kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersVerificationSecret == nil { kubeControllersVerificationSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, + Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, }, }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(enterprise.ElasticsearchKubeControllersUserName), "password": hashedPassword, }, } } - kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) + kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, enterprise.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersSecureUserSecret == nil { kubeControllersSecureUserSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, + Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index c51cbc076a..b8f056159e 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -44,6 +44,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" @@ -140,7 +141,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err := utils.AddDeploymentWatch(c, esgateway.DeploymentName, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } - if err := utils.AddDeploymentWatch(c, kubecontrollers.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { + if err := utils.AddDeploymentWatch(c, enterprise.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } @@ -168,7 +169,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { // Start goroutines to establish watches against projectcalico.org/v3 resources. go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, r.tierWatchReady) go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ - {Name: kubecontrollers.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, + {Name: enterprise.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, }) return nil @@ -262,7 +263,7 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec // Get secrets needed for kube-controllers to talk to elastic. This is needed for zero-tenants and single-tenants // that deploy es-kube-controllers and need to talk to es-gateway var kubeControllersUserSecret *core.Secret - kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, kubecontrollers.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) + kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, enterprise.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get kube controllers gateway secret", err, reqLogger) return reconcile.Result{}, err @@ -338,13 +339,12 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec ClusterDomain: r.clusterDomain, Authentication: authentication, KubeControllersGatewaySecret: kubeControllersUserSecret, - LogStorageExists: logStorage != nil, TrustedBundle: trustedBundle, Namespace: helper.InstallNamespace(), BindingNamespaces: namespaces, Tenant: nil, } - esKubeControllerComponents := kubecontrollers.NewElasticsearchKubeControllers(&kubeControllersCfg) + esKubeControllerComponents := enterprise.NewElasticsearchKubeControllers(&kubeControllersCfg) imageSet, err := imageset.GetImageSet(ctx, r.client, variant) if err != nil { diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go index b35d072664..c053617282 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go @@ -46,8 +46,8 @@ import ( "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/logstorage" "github.com/tigera/operator/pkg/render/logstorage/esgateway" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -235,7 +235,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: enterprise.EsKubeController, Namespace: common.CalicoNamespace, }, } @@ -275,12 +275,12 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: enterprise.EsKubeController, Namespace: common.CalicoNamespace, }, } Expect(test.GetResource(cli, &dep)).To(BeNil()) - kc := test.GetContainer(dep.Spec.Template.Spec.Containers, kubecontrollers.EsKubeController) + kc := test.GetContainer(dep.Spec.Template.Spec.Containers, enterprise.EsKubeController) Expect(kc).ToNot(BeNil()) Expect(kc.Image).To(Equal(fmt.Sprintf("some.registry.org/%s%s@%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, "sha256:kubecontrollershash"))) }) @@ -325,7 +325,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: enterprise.EsKubeController, Namespace: common.CalicoNamespace, }, } diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 14acd47230..404c399378 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -20,6 +20,7 @@ import ( v1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/common/discovery" + "github.com/tigera/operator/pkg/extensions" "k8s.io/client-go/kubernetes" ) @@ -55,4 +56,10 @@ type ControllerOptions struct { // the operator cares about. Populated once at startup so controllers can branch on API // availability without issuing further discovery requests at reconcile time. APIDiscovery *discovery.APIDiscovery + + // Extensions are the variant extensions (modifiers, image overrides, setups) + // the operator runs with. The core operator leaves this nil; an extension + // build (Calico Enterprise) sets it once at startup and controllers thread it + // into their render handlers and component configs. + Extensions *extensions.Set } diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index f330ea3028..b8f97b1fc3 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -47,6 +47,7 @@ import ( "github.com/tigera/operator/pkg/apigroup" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -74,16 +75,36 @@ type ComponentHandler interface { SetCreateOnly() } +// ComponentHandlerOption configures a componentHandler. +type ComponentHandlerOption func(*componentHandler) + +// WithRenderContext supplies the extensions.RenderContext passed to registered +// render modifiers. +func WithRenderContext(ctx extensions.RenderContext) ComponentHandlerOption { + return func(c *componentHandler) { c.renderCtx = ctx } +} + +// WithExtensions supplies the operator's extension Set, whose modifiers the +// handler applies to extensible components. A handler that renders an +// extensible component must be given the Set; one that doesn't can omit it. +func WithExtensions(e *extensions.Set) ComponentHandlerOption { + return func(c *componentHandler) { c.extensions = e } +} + // cr is allowed to be nil in the case we don't want to put ownership on a resource, // this is useful for CRD management so that they are not removed automatically. -func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object) ComponentHandler { - return &componentHandler{ +func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...ComponentHandlerOption) ComponentHandler { + h := &componentHandler{ client: cli, scheme: scheme, cr: cr, log: log, apiGroupEnvs: apigroup.EnvVars(), } + for _, o := range opts { + o(h) + } + return h } type componentHandler struct { @@ -93,6 +114,8 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar + renderCtx extensions.RenderContext + extensions *extensions.Set } func (c *componentHandler) SetCreateOnly() { @@ -435,6 +458,11 @@ func resetMetadataForCreate(obj client.Object) { } func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component render.Component, status status.StatusManager) error { + if ext, ok := component.(render.Extensible); ok && ext.ModifierKey() != "" && c.extensions == nil { + c.log.Info("BUG: extensible component rendered by a handler with no extension Set; extensions will not be applied", "component", ext.ModifierKey()) + } + component = c.extensions.Decorate(component, c.renderCtx) + // Before creating the component, make sure that it is ready. This provides a hook to do // dependency checking for the component. cmpLog := c.log.WithValues("component", reflect.TypeOf(component)) @@ -1139,7 +1167,6 @@ func addComponentLabel(obj metav1.Object, cr metav1.Object) { owner, ok := cr.(runtime.Object) if ok && owner.GetObjectKind() != nil && owner.GetObjectKind() != nil { obj.GetLabels()["app.kubernetes.io/component"] = sanitizeLabel(owner.GetObjectKind().GroupVersionKind().GroupKind().String()) - } } } diff --git a/pkg/controller/utils/component_enterprise_test.go b/pkg/controller/utils/component_enterprise_test.go new file mode 100644 index 0000000000..12d63fa44c --- /dev/null +++ b/pkg/controller/utils/component_enterprise_test.go @@ -0,0 +1,84 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +// This exercises the full path comment-by-comment: a real render component goes +// through CreateOrUpdateOrDelete with an enterprise RenderContext, and the +// registered modifier must match the real render output by name. If render ever +// renames the typha ClusterRole, the modifier silently no-ops and this fails. +var _ = Describe("componentHandler enterprise modifier integration", func() { + It("applies the enterprise typha modifier to real render output", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + nodeKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + typhaKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.TyphaCommonName}) + Expect(err).NotTo(HaveOccurred()) + + instance := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginCalico}, + } + comp := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + TLS: &render.TyphaNodeTLS{ + TrustedBundle: certManager.CreateTrustedBundle(), + TyphaSecret: typhaKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + }, + }) + + renderCtx := extensions.RenderContext{Installation: instance} + handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithRenderContext(renderCtx), utils.WithExtensions(enterprise.New())) + Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) + + role := &rbacv1.ClusterRole{} + Expect(cli.Get(context.Background(), client.ObjectKey{Name: "calico-typha"}, role)).NotTo(HaveOccurred()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/controller/utils/component_test.go b/pkg/controller/utils/component_test.go index 5936ce98bd..776ef6b4a5 100644 --- a/pkg/controller/utils/component_test.go +++ b/pkg/controller/utils/component_test.go @@ -46,6 +46,7 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -2575,3 +2576,45 @@ func (mc *mockClient) RESTMapper() restMeta.RESTMapper { func (mc *mockClient) SubResource(subResource string) client.SubResourceClient { panic("SubResource not implemented in mockClient") } + +var _ = Describe("componentHandler modifier application", func() { + It("applies registered modifiers to a named component before create", func() { + ext := extensions.NewSet() + ext.Variant(operatorv1.CalicoEnterprise).Modify("fake", func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + cm := objs[0].(*corev1.ConfigMap) + cm.Data = map[string]string{"patched": "yes"} + return objs, del + }) + + s := runtime.NewScheme() + Expect(apis.AddToScheme(s, false)).NotTo(HaveOccurred()) + Expect(corev1.SchemeBuilder.AddToScheme(s)).NotTo(HaveOccurred()) + + c := ctrlrfake.DefaultFakeClientBuilder(s).Build() + renderCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + handler := NewComponentHandler(logf.Log, c, s, nil, WithRenderContext(renderCtx), WithExtensions(ext)) + comp := &namedFakeComponent{name: "fake", obj: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "default"}, + }} + + Expect(handler.CreateOrUpdateOrDelete(context.Background(), comp, nil)).NotTo(HaveOccurred()) + + got := &corev1.ConfigMap{} + Expect(c.Get(context.Background(), client.ObjectKey{Name: "cm", Namespace: "default"}, got)).NotTo(HaveOccurred()) + Expect(got.Data).To(HaveKeyWithValue("patched", "yes")) + }) +}) + +type namedFakeComponent struct { + name string + obj client.Object +} + +func (f *namedFakeComponent) ModifierKey() string { return f.name } +func (f *namedFakeComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (f *namedFakeComponent) Objects() ([]client.Object, []client.Object) { + return []client.Object{f.obj}, nil +} +func (f *namedFakeComponent) Ready() bool { return true } +func (f *namedFakeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } diff --git a/pkg/enterprise/apiserver.go b/pkg/enterprise/apiserver.go new file mode 100644 index 0000000000..b20f3579e5 --- /dev/null +++ b/pkg/enterprise/apiserver.go @@ -0,0 +1,1147 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "fmt" + "slices" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rcomp "github.com/tigera/operator/pkg/render/common/components" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/securitycontext" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +const ( + auditLogsVolumeName = "calico-audit-logs" + auditPolicyVolumeName = "calico-audit-policy" +) + +// apiServer carries the rendered API server configuration and resolved image so the +// enterprise builders (moved verbatim from the render package) can construct the +// Enterprise-only objects and deployment additions. +type apiServer struct { + cfg *render.APIServerConfiguration + calicoImage string +} + +func registerAPIServer(v *extensions.Variant) { + extensions.RegisterModifier(v, render.ComponentNameAPIServer, modifyAPIServer) +} + +// registerAPIServerCleanup registers, for the Calico variant, the cleanup that +// deletes the Enterprise API server objects left behind by a prior Enterprise +// installation. +func registerAPIServerCleanup(v *extensions.Variant) { + extensions.RegisterModifier(v, render.ComponentNameAPIServer, cleanupAPIServer) +} + +// modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: +// the query server container and its volumes, audit logging on the aggregation API server +// container, the Enterprise RBAC objects, and the query server port on the Service. +func modifyAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { + c := &apiServer{cfg: ec.Config, calicoImage: ec.CalicoImage} + + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + c.layerDeployment(dep) + } + if svc, ok := extensions.FindObject[*corev1.Service](create, render.APIServerServiceName); ok { + c.addQueryServerPort(svc) + } + // Enterprise serves staged policies through the tiered-policy passthrough role. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](create, "calico-tiered-policy-passthrough"); ok { + for i := range role.Rules { + if slices.Contains(role.Rules[i].Resources, "networkpolicies") { + role.Rules[i].Resources = append(role.Rules[i].Resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") + } + } + } + + // Global Enterprise RBAC. + create = append(create, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + if !c.cfg.MultiTenant { + // These resources are only installed in zero-tenant clusters. + create = append(create, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + } + if c.cfg.ManagementCluster != nil { + create = append(create, c.managedClusterWatchClusterRole()) + if c.cfg.MultiTenant { + create = append(create, c.multiTenantSecretsRBAC()...) + create = append(create, c.multiTenantManagedClusterAccessClusterRoles()...) + } else { + create = append(create, c.secretsRBAC()...) + } + } else { + // If we're not a management cluster, the API server doesn't need permissions to access secrets. + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + } + + // Namespaced Enterprise objects. + if c.cfg.TrustedBundle != nil { + create = append(create, c.cfg.TrustedBundle.ConfigMap(render.QueryserverNamespace)) + } + if c.cfg.ManagementClusterConnection != nil { + create = append(create, c.externalLinseedRoleBinding()) + } + + // Objects that only exist alongside the aggregation API server. + aggregationObjects := []client.Object{ + c.uiSettingsGroupGetterClusterRole(), + c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), + c.uiSettingsPassthruClusterRole(), + c.uiSettingsPassthruClusterRolebinding(), + c.auditPolicyConfigMap(), + } + if c.cfg.RequiresAggregationServer { + create = append(create, aggregationObjects...) + } else { + del = append(del, aggregationObjects...) + } + + // Clean up cluster-scoped resources that were created with the 'tigera' prefix. + del = append(del, c.deprecatedResources()...) + + // Re-apply deployment overrides so the modifier-added query server container picks up + // any per-container overrides. The override appliers use replace/merge semantics, so + // re-running over the render-applied containers is idempotent. + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { + rcomp.ApplyDeploymentOverrides(dep, overrides) + } + } + + return create, del +} + +// cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a +// cluster switched from Enterprise to Calico does not leave them behind. +func cleanupAPIServer(rc extensions.RenderContext, ec render.APIServerExtensionContext, create, del []client.Object) ([]client.Object, []client.Object) { + c := &apiServer{cfg: ec.Config} + + del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + if !c.cfg.MultiTenant { + del = append(del, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + } + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + + return create, del +} + +// layerDeployment adds the Enterprise query server container, audit logging, and the +// query server / trusted bundle volumes to the rendered API server deployment. +func (c *apiServer) layerDeployment(d *appsv1.Deployment) { + // Audit logging is performed through the aggregation API server container, which is + // only present when the aggregation API server is running. + if c.cfg.RequiresAggregationServer { + for i := range d.Spec.Template.Spec.Containers { + ctr := &d.Spec.Template.Spec.Containers[i] + if ctr.Name != string(render.APIServerContainerName) { + continue + } + ctr.VolumeMounts = append(ctr.VolumeMounts, + corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, + corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, + ) + ctr.Args = append(ctr.Args, + "--audit-policy-file=/etc/tigera/audit/policy.conf", + "--audit-log-path=/var/log/calico/audit/tsee-audit.log", + ) + // In case of OpenShift, apiserver needs privileged access to write audit logs to the + // host path volume. Audit logs are owned by root on hosts so we need to be root. + ctr.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) + } + + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, c.auditVolumes()...) + } + + d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, c.queryServerContainer()) + + if c.cfg.TrustedBundle != nil { + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, c.cfg.TrustedBundle.Volume()) + for k, v := range c.cfg.TrustedBundle.HashAnnotations() { + d.Spec.Template.Annotations[k] = v + } + } +} + +// auditVolumes are the host-path audit log and audit policy volumes used by the +// aggregation API server container. +func (c *apiServer) auditVolumes() []corev1.Volume { + return []corev1.Volume{ + { + Name: auditLogsVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/log/calico/audit", + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + }, + { + Name: auditPolicyVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, + Items: []corev1.KeyToPath{ + { + Key: "config", + Path: "policy.conf", + }, + }, + }, + }, + }, + } +} + +func (c *apiServer) addQueryServerPort(s *corev1.Service) { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName) + s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ + Name: render.QueryServerPortName, + Port: render.QueryServerPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), + }) +} + +func (c *apiServer) multiTenantSecretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, true) +} + +func (c *apiServer) secretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.cfg.ManagementCluster, false) +} + +func (c *apiServer) queryServerContainer() corev1.Container { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName).ContainerPort + + var tlsSecret certificatemanagement.KeyPairInterface + if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { + tlsSecret = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly + } else { + tlsSecret = c.cfg.TLSKeyPair + } + env := []corev1.EnvVar{ + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, + {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) + } + + if render.HostNetwork(c.cfg) { + env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) + } else { + env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) + } + + if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if c.cfg.KeyValidatorConfig != nil { + env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) + } + + linseedURL := relasticsearch.LinseedEndpoint(rmeta.OSTypeLinux, c.cfg.ClusterDomain, render.ElasticsearchNamespace, c.cfg.ManagementClusterConnection != nil, false) + env = append(env, + corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, + corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + ) + if c.cfg.ManagementClusterConnection != nil { + env = append(env, + corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, + corev1.EnvVar{Name: "LINSEED_TOKEN", Value: render.GetLinseedTokenPath(true)}, + ) + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) + } + + // set LogLEVEL for queryserver container + if logging := c.cfg.APIServer.Logging; logging != nil && + logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { + env = append(env, + corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) + } else { + // set default LOGLEVEL to info when not set by the user + env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) + } + + volumeMounts := []corev1.VolumeMount{ + tlsSecret.VolumeMount(rmeta.OSTypeLinux), + } + if c.cfg.TrustedBundle != nil { + volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(rmeta.OSTypeLinux)...) + } + if c.cfg.ManagementClusterConnection != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: render.LinseedTokenVolumeName, + MountPath: render.LinseedVolumeMountPath, + }) + } + + container := corev1.Container{ + Name: string(render.TigeraAPIServerQueryServerContainerName), + Image: c.calicoImage, + Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, + Env: env, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/version", + Port: intstr.FromInt32(queryServerTargetPort), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 90, + }, + SecurityContext: securitycontext.NewNonRootContext(), + VolumeMounts: volumeMounts, + } + return container +} + +func (c *apiServer) externalLinseedRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-linseed", + Namespace: render.APIServerNamespace, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: render.TigeraLinseedSecretsClusterRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +func (c *apiServer) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + { + // Read access to Linseed policy activity data for queryserver enrichment. + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{"policyactivity"}, + Verbs: []string{"get"}, + }, + { + // Calico Enterprise backing storage. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "bfdconfigurations", + "deeppacketinspections", + "deeppacketinspections/status", + "egressgatewaypolicies", + "externalnetworks", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalreports", + "globalreports/status", + "globalreporttypes", + "globalthreatfeeds", + "globalthreatfeeds/status", + "licensekeys", + "managedclusters", + "managedclusters/status", + "networks", + "packetcaptures", + "packetcaptures/status", + "policyrecommendationscopes", + "policyrecommendationscopes/status", + "remoteclusterconfigurations", + "securityeventwebhooks", + "securityeventwebhooks/status", + "uisettings", + "uisettingsgroups", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "update", + "delete", + "patch", + }, + }, + { + // The queryserver's RBAC calculator needs to list tiers, + // uisettingsgroups, and managedclusters via the aggregated + // API to evaluate user permissions for the /policies endpoint. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "tiers", + "uisettingsgroups", + "managedclusters", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Required by the AuthorizationReview calculator in queryserver to evaluate + // RBAC permissions for users. + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{ + "clusterroles", + "clusterrolebindings", + "roles", + "rolebindings", + }, + Verbs: []string{"get", "list", "watch"}, + }, + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.APIServerServiceAccountName, + Namespace: render.APIServerNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: render.APIServerName, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "uisettingsgroups", + }, + Verbs: []string{"get"}, + }, + }, + } +} + +func (c *apiServer) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettingsgroup-getter", + APIGroup: "rbac.authorization.k8s.io", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "system:kube-controller-manager", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + } +} + +func (c *apiServer) tigeraUserClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // List requests that the Tigera manager needs. + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + "", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "namespaces", + "globalnetworksets", + "networksets", + "managedclusters", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "policyrecommendationscopes", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to view Networks. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Access to policies in all tiers + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"get"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // List and view the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"get", "watch", "list"}, + }, + // User can: + // - read UISettings in the cluster-settings group + // - read and write UISettings in the user-settings group + // Default settings group and settings are created in manager.go. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"get", "list", "watch"}, + ResourceNames: []string{"cluster-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"user-settings"}, + }, + // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read services to view WAF configuration. + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to only view securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list"}, + }, + } + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.cfg.ManagementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics. + // Access to log into Kibana for oidc users. + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: []string{ + "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", + }, + Verbs: []string{"get"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-ui-user", + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // Full access to all network policies + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "globalnetworksets", + "networksets", + "managedclusters", + "packetcaptures", + "policyrecommendationscopes", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{ + "policy.networking.k8s.io", + }, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get", "delete"}, + }, + // Allow the user to CRUD Networks. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + // Additional "list" requests that the Tigera Secure manager needs + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"watch", "list"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports/status"}, + Verbs: []string{"get", "list", "watch"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + // Access to cluster information containing Calico and EE versions from the UI. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // Manage the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + // User can: + // - read and write UISettings in the cluster-settings group, and rename the group + // - read and write UISettings in the user-settings group, and rename the group + // Default settings group and settings are created in manager.go. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get", "patch", "update"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + // Allow the user to read and write applicationlayers to enable/disable WAF. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get", "update", "patch", "create", "delete"}, + }, + // Allow the user to read deployments to view WAF configuration. + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to perform CRUD operations on securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, + }, + // Allow the user to create secrets. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + Verbs: []string{"create"}, + }, + // Allow the user to patch webhooks-secret secret. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + ResourceNames: []string{ + "webhooks-secret", + }, + Verbs: []string{"patch"}, + }, + } + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.cfg.ManagementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics. + // Elasticsearch superuser access once logged into Kibana. + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: []string{ + "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", + }, + Verbs: []string{"get"}, + }) + } + + // In v3 CRD / webhooks mode there is no aggregated apiserver, and the + // calico-uisettings-passthrough ClusterRole that normally grants the broad + // uisettings permission isn't deployed. Grant write verbs here so the + // calico-webhooks UISettings handler (which narrows access via a SAR on + // uisettingsgroups/data) gets invoked instead of being short-circuited by + // kube-apiserver RBAC. + if !c.cfg.RequiresAggregationServer { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"create", "update", "delete", "patch"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-network-admin", + }, + Rules: rules, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"*"}, + }, + }, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "Group", + Name: "system:authenticated", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettings-passthrough", + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) auditPolicyConfigMap() *corev1.ConfigMap { + const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: RequestResponse + omitStages: + - RequestReceived + verbs: + - create + - patch + - update + - delete + resources: + - group: projectcalico.org + resources: + - globalnetworkpolicies + - networkpolicies + - stagedglobalnetworkpolicies + - stagednetworkpolicies + - stagedkubernetesnetworkpolicies + - globalnetworksets + - networksets + - tiers + - hostendpoints` + + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + // This object is for Enterprise only, so pass it explicitly. + Namespace: render.APIServerNamespace, + Name: auditPolicyVolumeName, + }, + Data: map[string]string{ + "config": defaultAuditPolicy, + }, + } +} + +func (c *apiServer) multiTenantManagedClusterAccessClusterRoles() []client.Object { + var objects []client.Object + objects = append(objects, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.MultiTenantManagedClustersAccessClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for + // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. + // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. + "get", + }, + }, + }, + }) + + return objects +} + +func (c *apiServer) managedClusterWatchClusterRole() client.Object { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + "get", "list", "watch", + }, + }, + }, + } +} + +func (c *apiServer) deprecatedResources() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + + // delegateAuthClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, + }, + + // authClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + + // authClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + // authReaderRoleBinding - need clean up in diff namespace kube-system + &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-auth-reader", + Namespace: "kube-system", + }, + }, + // webhookReaderClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, + }, + + // webhookReaderClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, + }, + + // calico-apiserver CR and CRB + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + + // Clean up legacy secrets in the tigera-operator namespace + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, + }, + } +} diff --git a/pkg/enterprise/decorate_helpers_test.go b/pkg/enterprise/decorate_helpers_test.go new file mode 100644 index 0000000000..bf9c6a72d7 --- /dev/null +++ b/pkg/enterprise/decorate_helpers_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// stubExtComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate, the same seam the component +// handler uses. key selects the extension; extCtx is delivered as the component's +// ExtensionContext (the typed config a RegisterModifier modifier reads). +type stubExtComponent struct { + key string + extCtx any + create, delete []client.Object +} + +func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { + return s.create, s.delete +} + +func (s stubExtComponent) Ready() bool { + return true +} + +func (s stubExtComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s stubExtComponent) ModifierKey() string { + return s.key +} + +func (s stubExtComponent) ExtensionContext() any { + return s.extCtx +} + +// applyExtensions decorates a stub component holding the given objects with the +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use applyExtensionsWithContext. +func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return applyExtensionsWithContext(s, key, rc, nil, create, del) +} + +// applyExtensionsWithContext is applyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} + return s.Decorate(stub, rc).Objects() +} diff --git a/pkg/enterprise/enterprise_suite_test.go b/pkg/enterprise/enterprise_suite_test.go new file mode 100644 index 0000000000..368107b5a0 --- /dev/null +++ b/pkg/enterprise/enterprise_suite_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set under test, shared across the suite. It is +// immutable once built and the specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() + +func TestEnterprise(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise Suite") +} diff --git a/pkg/enterprise/guardian.go b/pkg/enterprise/guardian.go new file mode 100644 index 0000000000..4886889a29 --- /dev/null +++ b/pkg/enterprise/guardian.go @@ -0,0 +1,648 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "net" + "net/url" + + "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" + operatorurl "github.com/tigera/operator/pkg/url" +) + +func registerGuardian(v *extensions.Variant) { + extensions.RegisterModifier(v, render.GuardianName, modifyGuardian) + extensions.RegisterModifier(v, render.ComponentNameGuardianPolicy, modifyGuardianPolicy) +} + +// modifyGuardianPolicy replaces the core OSS guardian network policy with the +// enterprise management-cluster policy. Building the enterprise egress rules can +// fail (proxy URL parsing); on failure we drop the policy entirely, matching the +// core behavior of omitting it rather than installing a partial policy. +func modifyGuardianPolicy(rc extensions.RenderContext, gpc render.GuardianPolicyExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) + if !ok { + return objs, del + } + + spec, err := enterpriseGuardianPolicySpec(gpc) + if err != nil { + logrus.WithError(err).Error("Failed to build guardian network policy, policy will be omitted") + return removeObject(objs, policy), del + } + policy.Spec = spec + return objs, del +} + +func removeObject(objs []client.Object, drop client.Object) []client.Object { + out := objs[:0] + for _, o := range objs { + if o != drop { + out = append(out, o) + } + } + return out +} + +// enterpriseGuardianPolicySpec builds the network policy spec for guardian in a +// managed cluster: egress to the management cluster components and the tunnel +// destination(s), and ingress from the management-cluster components that reach +// back over the tunnel. +func enterpriseGuardianPolicySpec(gpc render.GuardianPolicyExtensionContext) (v3.NetworkPolicySpec, error) { + egressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.PacketCaptureEntityRule, + }, + } + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, gpc.OpenShift) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.KubeAPIServerEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.PrometheusEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.TigeraAPIServerEntityRule, + }, + }...) + + // Create an egress rule for each unique destination the guardian pods connect + // to. With multiple pods whose proxy settings differ, there are multiple + // destinations that must be allowed. + allowedDestinations := map[string]bool{} + for _, podProxyConfig := range render.ProcessPodProxies(gpc.PodProxies) { + var proxyURL *url.URL + var err error + if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { + // The scheme is HTTPS, as we establish an mTLS session with the target. + // We expect the URL to be of the form host:port. + targetURL := &url.URL{Scheme: "https", Host: gpc.URL} + proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + } + + var tunnelDestinationHostPort string + if proxyURL != nil { + proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + tunnelDestinationHostPort = proxyHostPort + } else { + // gpc.URL has host:port form. + tunnelDestinationHostPort = gpc.URL + } + + if allowedDestinations[tunnelDestinationHostPort] { + continue + } + + host, port, err := net.SplitHostPort(tunnelDestinationHostPort) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedPort, err := numorstring.PortFromString(port) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedIP := net.ParseIP(host) + if parsedIP == nil { + // Domain-based egress rules require the EgressAccessControl license feature. + if !gpc.IncludeEgressNetworkPolicy { + continue + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Domains: []string{host}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } else { + netSuffix := "/32" + if parsedIP.To4() == nil { + netSuffix = "/128" + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Nets: []string{parsedIP.String() + netSuffix}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } + } + + egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) + + dest := v3.EntityRule{Ports: networkpolicy.Ports(render.GuardianTargetPort)} + helper := networkpolicy.DefaultHelper() + ingressRules := []v3.Rule{ + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.FluentdSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceBenchmarkerSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceReporterSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceSnapshotterSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: helper.ComplianceControllerSourceEntityRule(), Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionInstallerSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Destination: dest}, + } + + return v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(render.GuardianName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, nil +} + +// modifyGuardian layers Calico Enterprise behavior onto the rendered guardian +// objects: the secrets Role/RoleBinding and default UI settings, the +// elasticsearch/kibana service ports, the management-cluster-request cluster +// role rules (which replace the OSS rules), and the CA bundle env vars. +func modifyGuardian(rc extensions.RenderContext, gc render.GuardianExtensionContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { + role.Rules = guardianEnterpriseRules(gc) + } + + if svc, ok := extensions.FindObject[*corev1.Service](objs, render.GuardianServiceName); ok { + svc.Spec.Ports = append(svc.Spec.Ports, guardianEnterpriseServicePorts()...) + } + + if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, render.GuardianDeploymentName); ok { + addGuardianEnterpriseEnv(gc, dep) + } + + return append(objs, + guardianSecretsRole(), + guardianSecretRoleBinding(), + // Default UI settings for this managed cluster. + render.ManagerClusterWideSettingsGroup(), + render.ManagerUserSpecificSettingsGroup(), + render.ManagerClusterWideTigeraLayer(), + render.ManagerClusterWideDefaultView(), + ), del +} + +// guardianEnterpriseRules are the cluster role rules guardian needs in Calico +// Enterprise. They wholly replace the OSS rules: the management cluster drives +// guardian over the tunnel, so it needs the union of the rules its components +// require, plus any configured impersonation and the OpenShift SCC. +func guardianEnterpriseRules(gc render.GuardianExtensionContext) []rbacv1.PolicyRule { + var rules []rbacv1.PolicyRule + + if imp := gc.Impersonation; imp != nil { + if imp.Users != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"users"}, + ResourceNames: imp.Users, + Verbs: []string{"impersonate"}, + }) + } + if imp.Groups != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"groups"}, + ResourceNames: imp.Groups, + Verbs: []string{"impersonate"}, + }) + } + if imp.ServiceAccounts != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + ResourceNames: imp.ServiceAccounts, + Verbs: []string{"impersonate"}, + }) + } + } + + rules = append(rules, rulesForManagementClusterRequests(gc.OpenShift)...) + + if gc.OpenShift { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.NonRootV2}, + }) + } + + return rules +} + +func guardianEnterpriseServicePorts() []corev1.ServicePort { + return []corev1.ServicePort{ + { + Name: "elasticsearch", + Port: 9200, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "kibana", + Port: 5601, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + } +} + +func addGuardianEnterpriseEnv(gc render.GuardianExtensionContext, dep *appsv1.Deployment) { + for i := range dep.Spec.Template.Spec.Containers { + c := &dep.Spec.Template.Spec.Containers[i] + if c.Name != render.GuardianContainerName { + continue + } + c.Env = append(c.Env, + corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + ) + } +} + +// guardianSecretsRole creates a Role that allows the management cluster to +// provision secrets to the tigera-operator Namespace, used to push secrets the +// managed cluster needs to access / authenticate with the management cluster. +func guardianSecretsRole() *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRole, + Namespace: common.OperatorNamespace(), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"create", "delete", "deletecollection", "update"}, + }, + }, + } +} + +func guardianSecretRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRoleBindingName, + Namespace: common.OperatorNamespace(), + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: render.GuardianSecretsRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +// rulesForManagementClusterRequests returns the set of RBAC rules guardian needs +// to satisfy requests from the management cluster over the tunnel. +func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + // Common rules required to handle requests from multiple components in the management cluster. + { + // ID uses read-only permissions and kube-controllers uses both read and write verbs. + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Allows Linseed to watch namespaces before copying its token. + // Also enables PolicyRecommendation to watch namespaces, + // and Manager/kube-controllers to list them. + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Nodes to monitor for deletions. + // Manager performs a list operation on Nodes. + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Pods to verify existence for IPAM garbage collection. + // Manager performs get operations on Pods. + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needed by kube-controllers to validate licenses; also used by ID. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation & ID uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworksets", + "networkpolicies", + "tier.networkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed by guardian to handle manager authorization reviews. + { + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, + Verbs: []string{"list", "get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings", "uisettingsgroups"}, + Verbs: []string{"list", "get"}, + }, + + // Rules needed by guardian to handle other manager requests. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + { + // Allow query server talk to Prometheus via the manager user. + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "calico-node-prometheus:9090", + "https:calico-api:8080", + }, + Verbs: []string{"create", "get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"daemonsets", "replicasets", "statefulsets"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"authentication.k8s.io"}, + Resources: []string{"tokenreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"authorization.k8s.io"}, + Resources: []string{"subjectaccessreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + ResourceNames: []string{"default"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworkpolicies", + "networksets", + "stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.globalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"list"}, + }, + + // Rules needed by guardian to handle policy recommendation requests. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "policyrecommendationscopes", + "policyrecommendationscopes/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + + // Rules needed by guardian to handle calico-kube-controller requests. + { + // Nodes are watched to monitor for deletions. + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services/status"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needs to manage hostendpoints. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Needs access to update clusterinformations. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Needs to manipulate kubecontrollersconfiguration, which contains its config. + // It creates a default if none exists, and updates status as well. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"kubecontrollersconfigurations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Grant permissions to access ClusterInformation resources in managed clusters. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + + // Rules needed by guardian to handle Intrusion detection requests. + { + APIGroups: []string{""}, + Resources: []string{"podtemplates"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalalerts", + "globalalerts/status", + "globalthreatfeeds", + "globalthreatfeeds/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed to fetch the compliance reports + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes", "globalreports"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + + // Rules needed by policy recommendation in openshift. + if isOpenShift { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, + }, + ) + } + + return rules +} diff --git a/pkg/enterprise/guardian_test.go b/pkg/enterprise/guardian_test.go new file mode 100644 index 0000000000..07e6694904 --- /dev/null +++ b/pkg/enterprise/guardian_test.go @@ -0,0 +1,101 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("guardian enterprise modifier", func() { + + // newObjs returns the subset of rendered guardian objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, Rules: []rbacv1.PolicyRule{{Verbs: []string{"get"}}}}, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "https", Port: 443}}}, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: render.GuardianContainerName}}, + }}}, + }, + } + } + + entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + + It("appends the secrets RBAC and UI settings", func() { + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) + _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*v3.UISettingsGroup](out, render.ManagerClusterSettings) + Expect(ok).To(BeTrue()) + }) + + It("adds the elasticsearch and kibana service ports", func() { + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, render.GuardianExtensionContext{}, newObjs(), nil) + svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) + names := []string{} + for _, p := range svc.Spec.Ports { + names = append(names, p.Name) + } + Expect(names).To(ContainElements("https", "elasticsearch", "kibana")) + }) + + It("replaces the cluster role rules and adds impersonation", func() { + gc := render.GuardianExtensionContext{ + Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, + } + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + + // The single OSS placeholder rule is gone, replaced by the enterprise set. + Expect(role.Rules).NotTo(ContainElement(rbacv1.PolicyRule{Verbs: []string{"get"}})) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"foo"})))) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"bar"})))) + }) + + It("adds the CA bundle env to the guardian container", func() { + gc := render.GuardianExtensionContext{TrustedBundleMountPath: "/ca/bundle"} + out, _ := applyExtensionsWithContext(ext, render.GuardianName, entCtx, gc, newObjs(), nil) + dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: "/ca/bundle"})) + }) + + It("does nothing for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out, _ := applyExtensions(ext, render.GuardianName, ctx, newObjs(), nil) + Expect(out).To(HaveLen(len(newObjs()))) + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) + }) +}) diff --git a/pkg/enterprise/installation.go b/pkg/enterprise/installation.go new file mode 100644 index 0000000000..7df2d2bf30 --- /dev/null +++ b/pkg/enterprise/installation.go @@ -0,0 +1,198 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "fmt" + + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// coreControllerExtension is the Calico Enterprise controller-side hook for the +// installation controller. +type coreControllerExtension struct{} + +// installationRenderData is the controller-produced data the installation +// extension hands to its modifiers through RenderContext.Extension. The node +// modifier type-asserts it back out. +type installationRenderData struct { + nodePrometheusTLS certificatemanagement.KeyPairInterface + + // kubeControllerTLS is the calico-kube-controllers metrics serving keypair; the + // kube-controllers modifier mounts it onto the deployment. + kubeControllerTLS certificatemanagement.KeyPairInterface + + // collectProcessPath mirrors LogCollector.Spec.CollectProcessPath being + // enabled; the node modifier uses it to set HostPID and the felix env. + collectProcessPath bool + + // calico-kube-controllers enterprise additions the kube-controllers modifier + // applies: the enterprise cluster role rules, the enterprise enabled controllers, + // and the WAF v3 (Gateway API add-on) surface. + kubeControllerRules []rbacv1.PolicyRule + kubeControllerControllers []string + waf wafRenderData +} + +// installationData pulls the installation extension's render data back out of the +// render context, returning the zero value when none is set. +func installationData(rc extensions.RenderContext) installationRenderData { + data, _ := rc.Extension.(installationRenderData) + return data +} + +func collectProcessPathEnabled(lc *operatorv1.LogCollector) bool { + return lc != nil && + lc.Spec.CollectProcessPath != nil && + *lc.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable +} + +// Validate rejects installation config Calico Enterprise does not support. +func (coreControllerExtension) Validate(cc extensions.ControllerContext) error { + return validateReporterPort(cc.FelixConfiguration) +} + +// Watches registers the enterprise resources the installation controller +// reconciles on. +func (coreControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, obj := range []client.Object{ + &operatorv1.ManagementCluster{}, + &operatorv1.ManagementClusterConnection{}, + &operatorv1.LogCollector{}, + // GatewayAPI.spec.extensions.waf.state gates the WAF v3 surface on calico-kube-controllers. + &operatorv1.GatewayAPI{}, + } { + if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + } + // es-kube-controllers includes the manager internal TLS secret in its bundle. + return utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) +} + +// ExtendContext does the controller-side work the modifiers can't: creating and +// fetching the certificates that feed the trusted bundle. It returns the render +// context carrying the produced node prometheus keypair, and that keypair as one +// the controller should manage. +func (coreControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { + rc := cc.RenderContext + + nodePrometheusTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return rc, nil, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + } + if nodePrometheusTLS != nil { + cc.TrustedBundle.AddCertificates(nodePrometheusTLS) + } + + // The calico-kube-controllers metrics endpoint is served with mTLS in + // Enterprise; the keypair is created here (cluster side effect) and mounted by + // the kube-controllers modifier. + kubeControllerTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + kubecontrollers.KubeControllerPrometheusTLSSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return rc, nil, fmt.Errorf("error creating kube-controllers metrics TLS certificate: %w", err) + } + if kubeControllerTLS != nil { + cc.TrustedBundle.AddCertificates(kubeControllerTLS) + } + + logCollector, err := utils.GetLogCollector(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error reading LogCollector: %w", err) + } + + // calico-kube-controllers enterprise additions: the WAF surface, the enterprise + // cluster role rules, and the enterprise enabled controllers. A managed cluster's + // kube-controllers needs an extra license-push rule. + managementClusterConnection, err := utils.GetManagementClusterConnection(cc.Ctx, cc.Client) + if err != nil { + return rc, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + waf, wafWebhookTLS, err := buildWAFData(cc) + if err != nil { + return rc, nil, fmt.Errorf("error preparing WAF configuration: %w", err) + } + + rc.Extension = installationRenderData{ + nodePrometheusTLS: nodePrometheusTLS, + kubeControllerTLS: kubeControllerTLS, + collectProcessPath: collectProcessPathEnabled(logCollector), + kubeControllerRules: calicoKubeControllersEnterpriseRules(waf.enabled, managementClusterConnection != nil), + kubeControllerControllers: calicoKubeControllersEnterpriseControllers(waf.enabled), + waf: waf, + } + + prometheusClientCert, err := cc.CertificateManager.GetCertificate(cc.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + if err != nil { + return rc, nil, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + } + if prometheusClientCert != nil { + cc.TrustedBundle.AddCertificates(prometheusClientCert) + } + + esgwCertificate, err := cc.CertificateManager.GetCertificate(cc.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + if err != nil { + return rc, nil, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + } + if esgwCertificate != nil { + cc.TrustedBundle.AddCertificates(esgwCertificate) + } + + // es-kube-controllers talks to Voltron, so the shared bundle must trust the + // manager internal cert. + managerInternalTLS, err := cc.CertificateManager.GetCertificate(cc.Client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) + if err != nil { + return rc, nil, fmt.Errorf("failed to retrieve %s: %w", render.ManagerInternalTLSSecretName, err) + } + if managerInternalTLS != nil { + cc.TrustedBundle.AddCertificates(managerInternalTLS) + } + + var managed []certificatemanagement.KeyPairInterface + if nodePrometheusTLS != nil { + managed = append(managed, nodePrometheusTLS) + } + if kubeControllerTLS != nil { + managed = append(managed, kubeControllerTLS) + } + if wafWebhookTLS != nil { + managed = append(managed, wafWebhookTLS) + } + return rc, managed, nil +} diff --git a/pkg/enterprise/installation_test.go b/pkg/enterprise/installation_test.go new file mode 100644 index 0000000000..d99e18eb00 --- /dev/null +++ b/pkg/enterprise/installation_test.go @@ -0,0 +1,84 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "k8s.io/apimachinery/pkg/runtime" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/kubecontrollers" +) + +var _ = Describe("installation controller extension", func() { + It("rejects a zero prometheus reporter port", func() { + port := 0 + cc := newControllerContext(operatorv1.CalicoEnterprise) + cc.FelixConfiguration = &v3.FelixConfiguration{ + Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, + } + Expect(ext.Validate(cc)).To(HaveOccurred()) + }) + + It("manages the node prometheus and kube-controllers metrics keypairs for the enterprise variant", func() { + _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + Expect(err).NotTo(HaveOccurred()) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ConsistOf(render.NodePrometheusTLSServerSecret, kubecontrollers.KubeControllerPrometheusTLSSecret)) + }) + + It("is a no-op for the Calico variant", func() { + _, managed, err := ext.ExtendContext(newControllerContext(operatorv1.Calico)) + Expect(err).NotTo(HaveOccurred()) + Expect(managed).To(BeEmpty()) + }) +}) + +func newControllerContext(variant operatorv1.ProductVariant) extensions.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + trustedBundle := certManager.CreateTrustedBundle() + + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: variant}, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: trustedBundle, + ClusterDomain: "cluster.local", + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/enterprise/kubecontrollers.go b/pkg/enterprise/kubecontrollers.go new file mode 100644 index 0000000000..2213e4c1d6 --- /dev/null +++ b/pkg/enterprise/kubecontrollers.go @@ -0,0 +1,714 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/gatewayapi" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/controller/utils/imageset" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/secret" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" + "github.com/tigera/operator/pkg/url" +) + +// registerKubeControllers registers the calico-kube-controllers modifiers. There is +// no image override: kube-controllers runs from the combined calico image, which +// resolves by variant in the base render. +func registerKubeControllers(v *extensions.Variant) { + v.Modify(render.ComponentNameKubeControllers, modifyKubeControllers) + v.Modify(render.ComponentNameKubeControllersPolicy, modifyKubeControllersPolicy) +} + +// modifyKubeControllersPolicy adds the WAF admission webhook ingress rule to the +// calico-kube-controllers calico-system network policy, so the kube-apiserver can +// reach the in-process webhook on :9443 (EV-6386). Without it the calico-system +// default-deny drops the apiserver->:9443 call and WAF admission times out. +func modifyKubeControllersPolicy(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if !installationData(rc).waf.enabled { + return objs, del + } + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) + if !ok { + return objs, del + } + policy.Spec.Ingress = append(policy.Spec.Ingress, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + }) + return objs, del +} + +// modifyKubeControllers layers the full Calico Enterprise surface onto the rendered +// calico-kube-controllers objects: the enterprise cluster role rules, the enterprise +// enabled controllers, the metrics serving TLS, and the WAF v3 (Gateway API add-on) +// surface. The modifier only runs for the enterprise variant, so everything it adds +// is enterprise-only by construction - the base render carries none of it. The +// controller-side inputs (keypairs, the resolved wasm image, the pull secret) are +// produced by the installation hook and handed in through rc. +func modifyKubeControllers(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + data := installationData(rc) + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole); ok { + role.Rules = append(role.Rules, data.kubeControllerRules...) + } + + if dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController); ok { + modifyKubeControllersDeployment(rc, dp, data) + } + + // The WAF admission webhook surface (Service + ValidatingWebhookConfiguration), + // the wasm pull secret, and the wasm CA bundle. Created when WAF is enabled, + // deleted otherwise so toggling the extension off cleans them up. + webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(data.waf.caBundle) + if data.waf.enabled { + objs = append(objs, webhookObjs...) + if data.waf.pullSecret != nil { + objs = append(objs, secret.ToRuntimeObjects(secret.CopyToNamespace(common.CalicoNamespace, data.waf.pullSecret)...)...) + } + if data.waf.caCert != nil { + objs = append(objs, data.waf.caCert) + } + } else { + del = append(del, webhookObjs...) + } + + return objs, del +} + +func modifyKubeControllersDeployment(rc extensions.RenderContext, dp *appsv1.Deployment, data installationRenderData) { + spec := &dp.Spec.Template.Spec + if dp.Spec.Template.Annotations == nil { + dp.Spec.Template.Annotations = map[string]string{} + } + + if tls := data.kubeControllerTLS; tls != nil { + spec.Volumes = append(spec.Volumes, tls.Volume()) + dp.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() + } + if waf := data.waf; waf.enabled && waf.webhookTLS != nil { + spec.Volumes = append(spec.Volumes, waf.webhookTLS.Volume()) + } + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != kubecontrollers.KubeController { + continue + } + + appendEnabledControllers(c, data.kubeControllerControllers) + c.Env = append(c.Env, enterpriseEnv(rc)...) + + if tls := data.kubeControllerTLS; tls != nil { + c.Env = append(c.Env, + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + ) + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if waf := data.waf; waf.enabled { + c.Env = append(c.Env, wafEnv(waf)...) + c.Ports = append(c.Ports, corev1.ContainerPort{ + Name: "waf-webhook", + ContainerPort: applicationlayer.WAFWebhookContainerPort, + Protocol: corev1.ProtocolTCP, + }) + if waf.webhookTLS != nil { + c.VolumeMounts = append(c.VolumeMounts, waf.webhookTLS.VolumeMount(rmeta.OSTypeLinux)) + if waf.webhookTLS.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, waf.webhookTLS.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + } + } +} + +// appendEnabledControllers folds the enterprise controllers into the existing +// ENABLED_CONTROLLERS env the base render set (node,loadbalancer). +func appendEnabledControllers(c *corev1.Container, extra []string) { + if len(extra) == 0 { + return + } + for i := range c.Env { + if c.Env[i].Name == "ENABLED_CONTROLLERS" { + c.Env[i].Value = c.Env[i].Value + "," + strings.Join(extra, ",") + return + } + } +} + +// enterpriseEnv is the static enterprise env for calico-kube-controllers. The +// modifier runs only for the enterprise variant, so these are never rendered for core. +func enterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { + var env []corev1.EnvVar + if rc.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: rc.TrustedBundle.MountPath()}) + } + if in := rc.Installation; in != nil && in.CalicoNetwork != nil && in.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: in.CalicoNetwork.MultiInterfaceMode.Value()}) + } + return env +} + +// wafEnv is the WAF v3 env the kube-controllers binary consumes to program WAF policy +// attachments. WASM_IMAGE is the pre-resolved reference the hook produced. +func wafEnv(waf wafRenderData) []corev1.EnvVar { + var env []corev1.EnvVar + if waf.wasmImage != "" { + env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: waf.wasmImage}) + } + if waf.pullSecret != nil { + env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: waf.pullSecret.Name}) + } + if waf.caCert != nil { + env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: waf.caCert.Name}) + } + if waf.webhookTLS != nil { + env = append(env, corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(waf.webhookTLS.VolumeMountCertificateFilePath())}) + } + return env +} + +const ( + EsKubeController = "es-calico-kube-controllers" + EsKubeControllerRole = "es-calico-kube-controllers" + EsKubeControllerRoleBinding = "es-calico-kube-controllers" + EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" + EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + + ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" + ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" + ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" + ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" + + // WASMPullSecretName is the dedicated image-pull Secret (a merged copy of the + // install pull secrets) the WAF reconciler replicates into tenant namespaces for + // the Coraza wasm OCI pull. A dedicated name avoids clashing with the + // operator-managed tigera-pull-secret the GatewayAPI render also copies there (EV-6386). + WASMPullSecretName = "tigera-waf-pull-secret" + + // WASMCACertName is the dedicated CA-bundle ConfigMap the WAF reconciler + // replicates into tenant namespaces for the Coraza wasm OCI registry TLS check - + // a dedicated name avoids clashing with the operator-managed tigera-ca-bundle the + // GatewayAPI render also copies there (EV-6386). It is a renamed copy of the trusted bundle. + WASMCACertName = "tigera-waf-ca-bundle" +) + +// NewElasticsearchKubeControllers fills the generic kube-controllers configuration +// for the enterprise es-calico-kube-controllers deployment and returns the rendered +// component. es-kube-controllers is a distinct deployment (talks to Elasticsearch via +// es-gateway) reconciled by the logstorage kube-controllers controller, so it's +// assembled here rather than through the render-time modifier mechanism. +func NewElasticsearchKubeControllers(cfg *kubecontrollers.KubeControllersConfiguration) render.Component { + cfg.Name = EsKubeController + cfg.ConfigName = "elasticsearch" + cfg.RoleName = EsKubeControllerRole + cfg.RoleBindingName = EsKubeControllerRoleBinding + cfg.MetricsName = EsKubeControllerMetrics + cfg.DisableConfigAPI = cfg.Tenant.MultiTenant() + + cfg.Rules = kubecontrollers.KubeControllersRoleCommonRules(cfg) + cfg.Rules = append(cfg.Rules, kubeControllersEnterpriseCommonRules(false, cfg.ManagementClusterConnection != nil)...) + cfg.Rules = append(cfg.Rules, + rbacv1.PolicyRule{ + APIGroups: []string{"elasticsearch.k8s.elastic.co"}, + Resources: []string{"elasticsearches"}, + Verbs: []string{"watch", "get", "list"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings"}, + Verbs: []string{"watch", "list", "get"}, + }, + ) + + if !cfg.Tenant.MultiTenant() { + // Zero and single tenant clusters need elasticsearch configuration. + cfg.EnabledControllers = append(cfg.EnabledControllers, "authorization", "elasticsearchconfiguration") + if cfg.ManagementCluster != nil && cfg.Tenant == nil { + // Enterprise requires the managedcluster controller to push licenses. + cfg.EnabledControllers = append(cfg.EnabledControllers, "managedcluster") + } + } + + cfg.NetworkPolicy = esKubeControllersCalicoSystemPolicy(cfg) + cfg.DeprecatedNetworkPolicyName = "es-kube-controller-access" + cfg.ExtraEnv = esKubeControllersEnv(cfg) + + return kubecontrollers.NewKubeControllers(cfg) +} + +// esKubeControllersEnv builds the enterprise env vars for es-calico-kube-controllers. +func esKubeControllersEnv(cfg *kubecontrollers.KubeControllersConfiguration) []corev1.EnvVar { + var env []corev1.EnvVar + + if cfg.Tenant != nil { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) + } + + // What started as a workaround is now the default behaviour. This feature uses our backend in order to + // log into Kibana for users from external identity providers, rather than configuring an authn realm + // in the Elastic stack. + env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) + if cfg.Authentication != nil { + env = append(env, + corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: cfg.Authentication.Spec.UsernamePrefix}, + corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: cfg.Authentication.Spec.GroupsPrefix}, + ) + } + + if cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) + } + if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if !cfg.Tenant.MultiTenant() { + _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(rmeta.OSTypeLinux, cfg.ClusterDomain, render.ElasticsearchNamespace)) + env = append(env, + relasticsearch.ElasticHostEnvVar(esHost), + relasticsearch.ElasticPortEnvVar(esPort), + relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticCAEnvVar(rmeta.OSTypeLinux), + ) + } + + return env +} + +// kubeControllersEnterpriseCommonRules are the Calico Enterprise cluster role rules +// shared by calico-kube-controllers and es-calico-kube-controllers. wafEnabled adds +// the WAF v3 (Gateway API add-on) rules; managedCluster adds the license-push rule a +// managed cluster's kube-controllers needs. +func kubeControllersEnterpriseCommonRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + }, + { + // Needed to validate the license + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + // Needed to update the status of the LicenseKey with the result of license validation. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures/status"}, + Verbs: []string{"update"}, + }, + } + + if wafEnabled { + rules = append(rules, wafRules()...) + } + + if managedCluster { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "create", "update", "list", "watch"}, + }, + ) + } + + return rules +} + +// calicoKubeControllersEnterpriseRules are the enterprise cluster role rules layered +// onto calico-kube-controllers: the shared enterprise rules plus the calico-specific +// ones (federated endpoints, license usage reporting). +func calicoKubeControllersEnterpriseRules(wafEnabled, managedCluster bool) []rbacv1.PolicyRule { + rules := kubeControllersEnterpriseCommonRules(wafEnabled, managedCluster) + return append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"watch", "list", "get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "update", "delete"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, + }, + ) +} + +// calicoKubeControllersEnterpriseControllers are the enterprise controllers added to +// the calico-kube-controllers ENABLED_CONTROLLERS list (on top of the base +// node,loadbalancer). applicationlayer is added only when the WAF extension is on. +func calicoKubeControllersEnterpriseControllers(wafEnabled bool) []string { + controllers := []string{"service", "federatedservices", "usage"} + if wafEnabled { + controllers = append(controllers, "applicationlayer") + } + return controllers +} + +// wafRules are the WAF v3 (Gateway API add-on) cluster role rules, gated by +// GatewayAPI.spec.extensions.waf.state == Enabled. +func wafRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + // Application-layer (gateway-addons) reconcilers reconcile WAF resources + // against Gateway API targetRefs and emit events on the policy objects. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies", "globalwafpolicies", + "wafplugins", "globalwafplugins", + "wafvalidationpolicies", "globalwafvalidationpolicies", + }, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/status", "globalwafpolicies/status", + "wafplugins/status", "globalwafplugins/status", + "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", + }, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/finalizers", "globalwafpolicies/finalizers", + "wafplugins/finalizers", "globalwafplugins/finalizers", + "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", + }, + Verbs: []string{"update"}, + }, + { + // Validate Gateway API targetRefs and surface attachment status. + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, + Verbs: []string{"get", "list", "watch", "update", "patch"}, + }, + { + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + // controller-runtime Reconcilers (e.g. the applicationlayer manager) record + // events on watched objects via Recorder.Eventf; both core and events.k8s.io + // API groups are emitted depending on the kubernetes version. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + { + APIGroups: []string{"events.k8s.io"}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + // Application-layer reconciler replicates the WAF wasm pull Secret from + // the controller namespace (calico-system) into each WAFPolicy's + // namespace so the rendered EnvoyExtensionPolicy can reference it. Also + // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. + { + APIGroups: []string{""}, + Resources: []string{"secrets", "configmaps"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF + // targetRef to bind the Coraza wasm filter at the gateway / route. + { + APIGroups: []string{"gateway.envoyproxy.io"}, + Resources: []string{"envoyextensionpolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler stamps each namespace with its allocated WAF + // rule-id range (applicationlayer.projectcalico.org/waf-id-range annotation) + // so application operators can author in-range rules. The base role already + // grants namespaces get/list/watch; the annotation write needs patch/update. + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "patch", "update"}, + }, + } +} + +func esKubeControllersCalicoSystemPolicy(cfg *kubecontrollers.KubeControllersConfiguration) *v3.NetworkPolicy { + if cfg.ManagementClusterConnection != nil { + return nil + } + + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(443, 6443, 12388), + }, + }, + }...) + + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), + }, + }...) + + networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicyHelper.ManagerEntityRule(), + }, + }...) + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EsKubeControllerNetworkPolicyName, + Namespace: cfg.Namespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(EsKubeController), + Types: []v3.PolicyType{v3.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} + +// wafRenderData is the controller-produced WAF v3 (Gateway API add-on) state the +// installation hook hands the kube-controllers modifier through the render context. +// The zero value (enabled false) means the modifier deletes the webhook objects. +type wafRenderData struct { + enabled bool + wasmImage string + pullSecret *corev1.Secret + caCert *corev1.ConfigMap + webhookTLS certificatemanagement.KeyPairInterface + caBundle []byte +} + +// buildWAFData reads the GatewayAPI CR and, when the WAF extension is enabled, +// produces everything the modifier needs that it can't compute itself: the resolved +// wasm image, the webhook serving keypair (also returned as a managed keypair), the +// merged wasm pull secret, the wasm CA bundle ConfigMap, and the operator CA PEM. +func buildWAFData(cc extensions.ControllerContext) (wafRenderData, certificatemanagement.KeyPairInterface, error) { + gw, _, err := gatewayapi.GetGatewayAPI(cc.Ctx, cc.Client) + if err != nil && !apierrors.IsNotFound(err) { + return wafRenderData{}, nil, err + } + if gw == nil || !gw.Spec.IsWAFGatewayExtensionEnabled() { + return wafRenderData{}, nil, nil + } + + in := cc.Installation + // The wasm is baked into the gateway envoy-proxy image. Resolve it with the same + // GetReference the base render uses for every image; the hook has the ImageSet here. + imageSet, err := imageset.GetImageSet(cc.Ctx, cc.Client, in.Variant) + if err != nil { + return wafRenderData{}, nil, err + } + wasmImage, err := components.GetReference(components.ComponentGatewayAPIEnvoyProxy, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return wafRenderData{}, nil, err + } + + webhookTLS, err := cc.CertificateManager.GetOrCreateKeyPair( + cc.Client, + applicationlayer.WAFWebhookServerTLSSecretName, + common.OperatorNamespace(), + dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return wafRenderData{}, nil, err + } + + pullSecrets, err := utils.GetInstallationPullSecrets(in, cc.Client) + if err != nil { + return wafRenderData{}, nil, err + } + var pullSecret *corev1.Secret + if len(pullSecrets) > 0 { + pullSecret, _ = MergeWAFPullSecret(pullSecrets) + } + + var caCert *corev1.ConfigMap + if cc.TrustedBundle != nil { + caCert = cc.TrustedBundle.ConfigMap(common.CalicoNamespace) + caCert.Name = WASMCACertName + } + + return wafRenderData{ + enabled: true, + wasmImage: wasmImage, + pullSecret: pullSecret, + caCert: caCert, + webhookTLS: webhookTLS, + caBundle: cc.CertificateManager.KeyPair().GetCertificatePEM(), + }, webhookTLS, nil +} + +// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret +// (tigera-waf-pull-secret) by merging the registry auths of every Installation pull +// secret. The EnvoyExtensionPolicy image source takes a single pullSecretRef, so a +// merged secret is the only way to honor multiple Installation pull secrets for the +// Coraza wasm OCI pull (e.g. the Tigera pull secret plus a private registry mirror). +// +// If the same registry appears in more than one secret, the first secret in +// Installation order wins. Secrets that cannot be parsed are skipped and their names +// returned, so the caller can log them without failing the reconcile. Returns a nil +// Secret when no registry auths could be collected. +func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { + merged := map[string]json.RawMessage{} + var skipped []string + for _, s := range pullSecrets { + auths, err := registryAuths(s) + if err != nil { + skipped = append(skipped, s.Name) + continue + } + for registry, auth := range auths { + if _, ok := merged[registry]; !ok { + merged[registry] = auth + } + } + } + if len(merged) == 0 { + return nil, skipped + } + + // Marshalling a map sorts its keys, so the rendered bytes are deterministic and + // do not churn the object on every reconcile. + data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) + if err != nil { + // Each auth entry round-trips from a successful Unmarshal above, so this + // cannot fail in practice; treat it as nothing to render. + return nil, skipped + } + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, + }, skipped +} + +// registryAuths extracts the per-registry auth entries from a pull secret of either +// the dockerconfigjson type (auths nested under an "auths" key) or the legacy +// dockercfg type (a bare registry -> auth map). +func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { + if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { + var cfg struct { + Auths map[string]json.RawMessage `json:"auths"` + } + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, err + } + if len(cfg.Auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return cfg.Auths, nil + } + if raw, ok := s.Data[corev1.DockerConfigKey]; ok { + var auths map[string]json.RawMessage + if err := json.Unmarshal(raw, &auths); err != nil { + return nil, err + } + if len(auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return auths, nil + } + return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) +} diff --git a/pkg/enterprise/kubecontrollers_test.go b/pkg/enterprise/kubecontrollers_test.go new file mode 100644 index 0000000000..3bb93c0e62 --- /dev/null +++ b/pkg/enterprise/kubecontrollers_test.go @@ -0,0 +1,292 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls" +) + +var _ = Describe("kube-controllers enterprise modifier", func() { + // kubeControllersDeployment is a minimal stand-in for the calico-kube-controllers + // deployment the base render produces, so the modifier has something to mount onto. + kubeControllersDeployment := func() *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.KubeController, Namespace: common.CalicoNamespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: kubecontrollers.KubeController}}, + }, + }, + }, + } + } + + It("mounts the metrics serving TLS keypair onto the deployment", func() { + rc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + c := dp.Spec.Template.Spec.Containers[0] + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + )) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Annotations).NotTo(BeEmpty(), "expected the cert hash annotation") + }) + + It("adds the cert-management init container when certificate management is enabled", func() { + rc, _, err := ext.ExtendContext(certManagementControllerContext()) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, []client.Object{kubeControllersDeployment()}, nil) + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.InitContainers[0].Name).To(Equal(fmt.Sprintf("%s-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) + }) +}) + +// certManagementControllerContext builds a controller context whose certificate +// manager issues cert-management (CSR-based) keypairs. +func certManagementControllerContext() extensions.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + ca, err := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) + Expect(err).NotTo(HaveOccurred()) + caCert, _, err := ca.Config.GetPEMBytes() + Expect(err).NotTo(HaveOccurred()) + + installation := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CertificateManagement: &operatorv1.CertificateManagement{CACert: caCert}, + } + certManager, err := certificatemanager.Create(c, installation, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: installation, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} + +var _ = Describe("calico-kube-controllers enterprise surface", func() { + calicoKubeControllersCfg := func(cc extensions.ControllerContext) *kubecontrollers.KubeControllersConfiguration { + return &kubecontrollers.KubeControllersConfiguration{ + Installation: cc.Installation, + ClusterDomain: cc.ClusterDomain, + TrustedBundle: cc.TrustedBundle, + MetricsPort: 9094, + Namespace: common.CalicoNamespace, + BindingNamespaces: []string{common.CalicoNamespace}, + } + } + + // render builds the base calico-kube-controllers objects and applies the + // enterprise modifier, exactly as the component handler does. + renderKubeControllers := func(cc extensions.ControllerContext, rc extensions.RenderContext) []client.Object { + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + out, _ := applyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + return out + } + + kubeContainer := func(objs []client.Object) *corev1.Container { + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + return &dp.Spec.Template.Spec.Containers[0] + } + + It("layers the enterprise rules, controllers, and metrics TLS on (WAF off)", func() { + rc, _, err := ext.ExtendContext(newControllerContext(operatorv1.CalicoEnterprise)) + Expect(err).NotTo(HaveOccurred()) + objs := renderKubeControllers(newControllerContext(operatorv1.CalicoEnterprise), rc) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage", + })) + // Metrics serving TLS wired from the keypair the hook created. + Expect(c.Env).To(ContainElement(HaveField("Name", "TLS_KEY_PATH"))) + // WAF is off, so no WASM env and no webhook objects. + Expect(c.Env).NotTo(ContainElement(HaveField("Name", "WASM_IMAGE"))) + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeFalse()) + }) + + It("layers the full WAF surface on when the GatewayAPI extension is enabled", func() { + cc := wafControllerContext() + rc, managed, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ContainElement(applicationlayer.WAFWebhookServerTLSSecretName)) + + objs := renderKubeControllers(cc, rc) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("wafpolicies")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: enterprise.WASMPullSecretName})) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_CA_CERT", Value: enterprise.WASMCACertName})) + Expect(c.Env).To(ContainElement(HaveField("Name", "WAF_WEBHOOK_CERT_DIR"))) + Expect(c.Ports).To(ContainElement(corev1.ContainerPort{Name: "waf-webhook", ContainerPort: int32(9443), Protocol: corev1.ProtocolTCP})) + + // The webhook surface, the wasm pull secret, and the wasm CA bundle are rendered. + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.Secret](objs, enterprise.WASMPullSecretName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.ConfigMap](objs, enterprise.WASMCACertName) + Expect(ok).To(BeTrue()) + }) + + It("deletes the WAF webhook surface when the extension is disabled", func() { + cc := newControllerContext(operatorv1.CalicoEnterprise) + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(cc)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + _, toDelete := applyExtensions(ext, render.ComponentNameKubeControllers, rc, create, del) + + _, ok := extensions.FindObject[*corev1.Service](toDelete, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue(), "the webhook Service should be queued for deletion") + }) + + It("adds the WAF webhook ingress rule to the network policy when enabled", func() { + cc := wafControllerContext() + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllersPolicy(calicoKubeControllersCfg(cc), nil) + create, del := comp.Objects() + objs, _ := applyExtensions(ext, render.ComponentNameKubeControllersPolicy, rc, create, del) + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) + Expect(ok).To(BeTrue()) + Expect(policy.Spec.Ingress).To(ContainElement(v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + })) + }) +}) + +// wafControllerContext builds a controller context with a WAF-enabled GatewayAPI CR +// and an install pull secret, so the installation hook produces the full WAF data. +func wafControllerContext() extensions.ControllerContext { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + Expect(c.Create(context.Background(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "pull", Namespace: common.OperatorNamespace()}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte(`{"auths":{"reg.example.com":{"auth":"abc"}}}`)}, + })).NotTo(HaveOccurred()) + + enabled := operatorv1.WAFExtensionStateEnabled + Expect(c.Create(context.Background(), &operatorv1.GatewayAPI{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: operatorv1.GatewayAPISpec{ + Extensions: &operatorv1.GatewayAPIExtensions{WAF: &operatorv1.WAFExtensionSpec{State: &enabled}}, + }, + })).NotTo(HaveOccurred()) + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "test-reg/", + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "pull"}}, + }, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/enterprise/node.go b/pkg/enterprise/node.go new file mode 100644 index 0000000000..d4e35dae68 --- /dev/null +++ b/pkg/enterprise/node.go @@ -0,0 +1,303 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "errors" + "fmt" + "slices" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +const ( + // defaultNodeReporterPort is the port calico/node reports Enterprise internal + // metrics on when FelixConfiguration does not override prometheusReporterPort. + defaultNodeReporterPort = 9081 + + // defaultFelixMetricsPort is the Felix prometheus metrics port used when + // FelixConfiguration does not override prometheusMetricsPort. + defaultFelixMetricsPort = 9091 + + installCNIContainerName = "install-cni" +) + +func registerNode(v *extensions.Variant) { + v.Image(render.ComponentNameNode, components.ComponentTigeraNode) + v.Modify(render.ComponentNameNode, modifyNode) + + // The node component renders the cni-plugins init container; its image + // resolves through its own override key. + v.Image(render.ComponentNameCNIPlugins, components.ComponentTigeraCNIPlugins) +} + +// modifyNode layers Calico Enterprise behavior onto the rendered calico/node +// objects: the extra RBAC rules, the node-metrics Service, and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics +// readiness check, multi-interface mode, and the calico log volume). +func modifyNode(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { + role.Rules = append(role.Rules, nodeEnterpriseRules()...) + } + + // The Network resource is only available in Enterprise / Cloud at this time. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"get"}, + }) + } + + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName); ok { + modifyNodeDaemonSet(rc, ds) + } + + return append(objs, nodeMetricsService(rc)), del +} + +// nodeEnterpriseRules are the additional cluster role rules calico/node needs in +// Calico Enterprise. +func nodeEnterpriseRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + // Calico Enterprise needs to be able to read additional resources. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Tigera Secure updates status for packet captures. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "packetcaptures", + "packetcaptures/status", + }, + Verbs: []string{"update"}, + }, + } +} + +// modifyNodeDaemonSet applies the Enterprise-specific daemonset changes that the +// base render leaves out: the Enterprise felix env, multi-interface mode, the +// BGP metrics readiness check, and the prometheus reporter keypair mount. The +// calico log volume is mounted by the base render for both variants, so it is +// not handled here. +func modifyNodeDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { + spec := &ds.Spec.Template.Spec + + // Collecting process info for flow logs reads from the host's process table. + if installationData(rc).collectProcessPath { + spec.HostPID = true + } + + multiInterfaceMode := multiInterfaceModeEnv(rc.Installation) + + for i := range spec.InitContainers { + if spec.InitContainers[i].Name == installCNIContainerName && multiInterfaceMode != nil { + spec.InitContainers[i].Env = append(spec.InitContainers[i].Env, *multiInterfaceMode) + } + } + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != render.CalicoNodeObjectName { + continue + } + + c.Env = append(c.Env, nodeEnterpriseEnv(rc)...) + + // Add the BGP metrics readiness check, but only when the base render kept + // the bird readiness check (i.e. BGP is in use and we're not on VPP). + if c.ReadinessProbe != nil && c.ReadinessProbe.Exec != nil && slices.Contains(c.ReadinessProbe.Exec.Command, "--bird-ready") { + c.ReadinessProbe.Exec.Command = append(c.ReadinessProbe.Exec.Command, "--bgp-metrics-ready") + } + } + + mountNodePrometheusTLS(rc, ds) +} + +// mountNodePrometheusTLS mounts the node prometheus reporter keypair onto the +// daemonset: the volume, the calico-node volume mount, the cert-management init +// container (when in use), and the pod hash annotation that rolls the pods on +// cert rotation. The keypair has cluster side effects, so the enterprise setup +// creates it and hands it in via rc rather than the modifier building it. In +// core (calico) the keypair is never created, so the base node render carries +// no prometheus mount at all. +func mountNodePrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { + tls := installationData(rc).nodePrometheusTLS + if tls == nil { + return + } + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if c.Name != render.CalicoNodeObjectName { + continue + } + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} + +// nodeEnterpriseEnv is the Enterprise felix configuration added to the +// calico/node container. +func nodeEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { + data := installationData(rc) + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if data.collectProcessPath { + env = append(env, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) + } + + if mode := multiInterfaceModeEnv(rc.Installation); mode != nil { + env = append(env, *mode) + } + + tls := data.nodePrometheusTLS + if tls != nil && rc.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, + ) + } + + return env +} + +// multiInterfaceModeEnv returns the MULTI_INTERFACE_MODE env var when the +// installation configures it, or nil otherwise. +func multiInterfaceModeEnv(install *operatorv1.InstallationSpec) *corev1.EnvVar { + if install.CalicoNetwork != nil && install.CalicoNetwork.MultiInterfaceMode != nil { + return &corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: install.CalicoNetwork.MultiInterfaceMode.Value()} + } + return nil +} + +// nodeMetricsService builds the enterprise-only calico-node-metrics Service. +func nodeMetricsService(rc extensions.RenderContext) *corev1.Service { + reporterPort := nodeReporterPort(rc.FelixConfiguration) + felixPort := felixMetricsPort(rc.FelixConfiguration) + felixEnabled := rc.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(rc.FelixConfiguration) + + ports := []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + } + if felixEnabled { + ports = append(ports, corev1.ServicePort{ + Name: "felix-metrics-port", + Port: int32(felixPort), + TargetPort: intstr.FromInt(felixPort), + Protocol: corev1.ProtocolTCP, + }) + } + + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.CalicoNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + ClusterIP: "None", + Ports: ports, + }, + } +} + +// validateReporterPort rejects the unsupported zero prometheus reporter port. +// The node and windows controller extensions share it. +func validateReporterPort(fc *v3.FelixConfiguration) error { + if fc != nil && fc.Spec.PrometheusReporterPort != nil && *fc.Spec.PrometheusReporterPort == 0 { + return errors.New("felixConfiguration prometheusReporterPort=0 not supported") + } + return nil +} + +// nodeReporterPort returns the reporter metrics port from the FelixConfiguration, +// falling back to the default. The node-metrics Service and the +// FELIX_PROMETHEUSREPORTERPORT env var both derive from here so they can't drift. +func nodeReporterPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusReporterPort != nil { + return *fc.Spec.PrometheusReporterPort + } + return defaultNodeReporterPort +} + +// felixMetricsPort returns the Felix prometheus metrics port from the +// FelixConfiguration, falling back to the default. +func felixMetricsPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusMetricsPort != nil { + return *fc.Spec.PrometheusMetricsPort + } + return defaultFelixMetricsPort +} diff --git a/pkg/enterprise/node_test.go b/pkg/enterprise/node_test.go new file mode 100644 index 0000000000..4257766122 --- /dev/null +++ b/pkg/enterprise/node_test.go @@ -0,0 +1,190 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("node enterprise image override", func() { + + It("selects the enterprise node image for the enterprise variant", func() { + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + Expect(ext.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + }) + + It("leaves the default in place for the Calico variant", func() { + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(ext.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) + }) +}) + +var _ = Describe("node enterprise modifier", func() { + + // newObjs returns the subset of rendered node objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoNodeObjectName}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoCNIPluginObjectName}}, + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "install-cni"}}, + Containers: []corev1.Container{{ + Name: render.CalicoNodeObjectName, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{ + Command: []string{"/bin/calico-node", "-bird-ready", "--bird-ready", "--felix-ready"}, + }}}, + }}, + }}}, + }, + } + } + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + entCtx := func() extensions.RenderContext { + return extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + } + + It("adds the enterprise cluster role rules", func() { + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ConsistOf("networks")))) + }) + + It("adds the enterprise felix env to the node container", func() { + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + c := nodeContainer(ds) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + }) + + It("derives the reporter port from FelixConfiguration", func() { + reporter := 7081 + ctx := entCtx() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} + + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) + }) + + It("appends the BGP metrics readiness check when the bird check is present", func() { + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) + }) + + It("does not add the BGP metrics readiness check when the bird check is absent", func() { + objs := newObjs() + ds := objs[2].(*appsv1.DaemonSet) + ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} + + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), objs, nil) + got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) + }) + + It("adds MULTI_INTERFACE_MODE to the node and install-cni containers when configured", func() { + mode := operatorv1.MultiInterfaceModeMultus + ctx := entCtx() + ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} + + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + + want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} + Expect(nodeContainer(ds).Env).To(ContainElement(want)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Env).To(ContainElement(want)) + }) + + It("appends the node metrics service", func() { + out, _ := applyExtensions(ext, render.ComponentNameNode, entCtx(), newObjs(), nil) + svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(HaveLen(2)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + Expect(svc.Spec.Ports[1].Port).To(Equal(int32(9900))) + }) + + It("derives metrics service ports and felix-metrics-port from FelixConfiguration", func() { + reporter := 7081 + metrics := 7091 + enabled := true + ctx := entCtx() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ + PrometheusReporterPort: &reporter, + PrometheusMetricsPort: &metrics, + PrometheusMetricsEnabled: &enabled, + }} + + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(svc.Spec.Ports).To(HaveLen(3)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) + Expect(svc.Spec.Ports[2].Name).To(Equal("felix-metrics-port")) + Expect(svc.Spec.Ports[2].Port).To(Equal(int32(7091))) + }) + + It("is a no-op for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out, _ := applyExtensions(ext, render.ComponentNameNode, ctx, newObjs(), nil) + + _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeFalse()) + nodeRole, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(nodeRole.Rules).To(BeEmpty()) + }) + + It("does not panic on a zero RenderContext", func() { + out, _ := applyExtensions(ext, render.ComponentNameNode, extensions.RenderContext{}, newObjs(), nil) + _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeFalse()) + }) +}) diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go new file mode 100644 index 0000000000..839e56b66a --- /dev/null +++ b/pkg/enterprise/register.go @@ -0,0 +1,46 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" +) + +// New builds the extension Set for the in-repo Calico Enterprise variant: the +// controller extension, every component modifier, and the image overrides. The +// operator is handed this Set at startup (the core operator is handed none). +// After the monorepo split this is what calico-private's main will construct +// instead. +func New() *extensions.Set { + s := extensions.NewSet() + + ent := s.Variant(operatorv1.CalicoEnterprise) + ent.Controller(extensions.InstallationController, coreControllerExtension{}) + ent.Controller(extensions.WindowsController, windowsControllerExtension{}) + registerTypha(ent) + registerNode(ent) + registerWindows(ent) + registerGuardian(ent) + registerAPIServer(ent) + registerKubeControllers(ent) + + // When the enterprise operator manages a Calico installation, clean up the + // Enterprise objects left behind by a prior Enterprise installation. + cal := s.Variant(operatorv1.Calico) + registerAPIServerCleanup(cal) + + return s +} diff --git a/pkg/enterprise/typha.go b/pkg/enterprise/typha.go new file mode 100644 index 0000000000..3786e7be07 --- /dev/null +++ b/pkg/enterprise/typha.go @@ -0,0 +1,62 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +func registerTypha(v *extensions.Variant) { + v.Modify(render.ComponentNameTypha, modifyTypha) +} + +func modifyTypha(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha"); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "deeppacketinspections", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }) + } + + if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, "calico-typha"); ok { + net := rc.Installation.CalicoNetwork + if net != nil && net.MultiInterfaceMode != nil { + for i := range dep.Spec.Template.Spec.Containers { + if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { + c := &dep.Spec.Template.Spec.Containers[i] + c.Env = append(c.Env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: net.MultiInterfaceMode.Value()}) + } + } + } + } + + return objs, del +} diff --git a/pkg/enterprise/typha_test.go b/pkg/enterprise/typha_test.go new file mode 100644 index 0000000000..cfc60b2b62 --- /dev/null +++ b/pkg/enterprise/typha_test.go @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("typha enterprise modifier", func() { + + multiMode := operatorv1.MultiInterfaceModeMultus + + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-typha"}}, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "calico-typha"}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: render.TyphaContainerName}}, + }}}, + }, + } + } + + It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + }} + out, _ := applyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) + + role := out[0].(*rbacv1.ClusterRole) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + dep := out[1].(*appsv1.Deployment) + var c *corev1.Container + for i := range dep.Spec.Template.Spec.Containers { + if dep.Spec.Template.Spec.Containers[i].Name == render.TyphaContainerName { + c = &dep.Spec.Template.Spec.Containers[i] + } + } + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: multiMode.Value()})) + }) + + It("is a no-op for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.Calico, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + }} + out, _ := applyExtensions(ext, render.ComponentNameTypha, ctx, newObjs(), nil) + Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) + dep := out[1].(*appsv1.Deployment) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(BeEmpty()) + }) + + It("does not panic on a zero Context (nil Installation)", func() { + out, _ := applyExtensions(ext, render.ComponentNameTypha, extensions.RenderContext{}, newObjs(), nil) + Expect(out[0].(*rbacv1.ClusterRole).Rules).To(BeEmpty()) + }) +}) diff --git a/pkg/render/kubecontrollers/waf_pull_secret_test.go b/pkg/enterprise/waf_pull_secret_test.go similarity index 88% rename from pkg/render/kubecontrollers/waf_pull_secret_test.go rename to pkg/enterprise/waf_pull_secret_test.go index 793374f169..92cab8df9d 100644 --- a/pkg/render/kubecontrollers/waf_pull_secret_test.go +++ b/pkg/enterprise/waf_pull_secret_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kubecontrollers_test +package enterprise_test import ( "encoding/json" @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/enterprise" ) func dockerConfigJSONSecret(name string, auths map[string]any) *corev1.Secret { @@ -49,7 +49,7 @@ func mergedAuths(t *testing.T, s *corev1.Secret) map[string]map[string]string { } func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("tigera-pull-secret", map[string]any{"quay.io": map[string]string{"auth": "dGlnZXJh"}}), dockerConfigJSONSecret("mirror-pull-secret", map[string]any{"registry.example.com": map[string]string{"auth": "bWlycm9y"}}), }) @@ -59,7 +59,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { if merged == nil { t.Fatal("expected a merged secret") } - if merged.Name != kubecontrollers.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { + if merged.Name != enterprise.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { t.Fatalf("unexpected name/namespace: %s/%s", merged.Namespace, merged.Name) } if merged.Type != corev1.SecretTypeDockerConfigJson { @@ -72,7 +72,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { } func TestMergeWAFPullSecret_FirstSecretWinsOnDuplicateRegistry(t *testing.T) { - merged, _ := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, _ := enterprise.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("first", map[string]any{"quay.io": map[string]string{"auth": "Zmlyc3Q="}}), dockerConfigJSONSecret("second", map[string]any{"quay.io": map[string]string{"auth": "c2Vjb25k"}}), }) @@ -88,7 +88,7 @@ func TestMergeWAFPullSecret_SkipsUnparseableSecrets(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{ bad, dockerConfigJSONSecret("good", map[string]any{"quay.io": map[string]string{"auth": "Z29vZA=="}}), }) @@ -111,7 +111,7 @@ func TestMergeWAFPullSecret_LegacyDockercfg(t *testing.T) { Type: corev1.SecretTypeDockercfg, Data: map[string][]byte{corev1.DockerConfigKey: cfg}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{legacy}) + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{legacy}) if len(skipped) != 0 { t.Fatalf("expected no skipped secrets, got %v", skipped) } @@ -127,7 +127,7 @@ func TestMergeWAFPullSecret_NothingUsableReturnsNil(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{bad}) + merged, skipped := enterprise.MergeWAFPullSecret([]*corev1.Secret{bad}) if merged != nil { t.Fatalf("expected nil secret, got %v", merged) } @@ -141,8 +141,8 @@ func TestMergeWAFPullSecret_DeterministicOutput(t *testing.T) { dockerConfigJSONSecret("a", map[string]any{"z.example.com": map[string]string{"auth": "eg=="}, "a.example.com": map[string]string{"auth": "YQ=="}}), dockerConfigJSONSecret("b", map[string]any{"m.example.com": map[string]string{"auth": "bQ=="}}), } - first, _ := kubecontrollers.MergeWAFPullSecret(in) - second, _ := kubecontrollers.MergeWAFPullSecret(in) + first, _ := enterprise.MergeWAFPullSecret(in) + second, _ := enterprise.MergeWAFPullSecret(in) if string(first.Data[corev1.DockerConfigJsonKey]) != string(second.Data[corev1.DockerConfigJsonKey]) { t.Fatal("merged secret bytes must be deterministic across reconciles") } diff --git a/pkg/enterprise/windows.go b/pkg/enterprise/windows.go new file mode 100644 index 0000000000..183bfd08ae --- /dev/null +++ b/pkg/enterprise/windows.go @@ -0,0 +1,240 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// windowsNodeContainers are the calico-node-windows containers that share the +// felix env and node volume mounts, so they receive the same enterprise layering. +var windowsNodeContainers = map[string]bool{"felix": true, "node": true, "confd": true} + +func registerWindows(v *extensions.Variant) { + v.Image(render.ComponentNameWindowsNodeImg, components.ComponentTigeraNodeWindows) + v.Image(render.ComponentNameWindowsCNIImg, components.ComponentTigeraCNIWindows) + v.Modify(render.ComponentNameWindows, modifyWindows) +} + +// windowsControllerExtension is the Calico Enterprise controller-side hook for the +// windows controller. +type windowsControllerExtension struct{} + +// windowsRenderData is the controller-produced data the windows extension hands to +// its modifier through RenderContext.Extension. +type windowsRenderData struct { + prometheusServerTLS certificatemanagement.KeyPairInterface +} + +// windowsData pulls the windows extension's render data back out of the render +// context, returning the zero value when none is set. +func windowsData(rc extensions.RenderContext) windowsRenderData { + data, _ := rc.Extension.(windowsRenderData) + return data +} + +// Validate rejects windows installation config Calico Enterprise does not support. +func (windowsControllerExtension) Validate(cc extensions.ControllerContext) error { + return validateReporterPort(cc.FelixConfiguration) +} + +// Watches registers the enterprise secrets the windows controller reconciles on. +func (windowsControllerExtension) Watches(c ctrlruntime.Controller) error { + for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { + if err := utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { + return err + } + if err := utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { + return err + } + } + return nil +} + +// ExtendContext fetches the node prometheus keypair the installation controller +// created and stashes it in the render context for the windows modifier. +func (windowsControllerExtension) ExtendContext(cc extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { + rc := cc.RenderContext + tls, err := cc.CertificateManager.GetKeyPair( + cc.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, cc.ClusterDomain), + ) + if err != nil { + return rc, nil, fmt.Errorf("error getting node prometheus TLS certificate: %w", err) + } + rc.Extension = windowsRenderData{prometheusServerTLS: tls} + return rc, nil, nil +} + +// modifyWindows layers Calico Enterprise behavior onto the rendered +// calico-node-windows objects: the node-metrics Service and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS +// servers, the calico log volume, and the prometheus reporter keypair mount). +func modifyWindows(rc extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { + modifyWindowsDaemonSet(rc, ds) + } + + return append(objs, windowsNodeMetricsService(rc)), del +} + +func modifyWindowsDaemonSet(rc extensions.RenderContext, ds *appsv1.DaemonSet) { + dirOrCreate := corev1.HostPathDirectoryOrCreate + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: "var-log-calico", + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, + }) + + for i := range spec.Containers { + c := &spec.Containers[i] + if !windowsNodeContainers[c.Name] { + continue + } + + c.Env = append(c.Env, windowsEnterpriseEnv(rc)...) + + // Enterprise mounts the calico log directory in place of the OSS CNI log + // directory, so drop the OSS mount before adding the enterprise one. + c.VolumeMounts = removeVolumeMount(c.VolumeMounts, "cni-log-dir") + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) + } + + mountWindowsPrometheusTLS(rc, ds) +} + +// windowsEnterpriseEnv is the Enterprise felix configuration added to the +// calico-node-windows containers. +func windowsEnterpriseEnv(rc extensions.RenderContext) []corev1.EnvVar { + tls := windowsData(rc).prometheusServerTLS + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", nodeReporterPort(rc.FelixConfiguration))}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if tls != nil && rc.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: rc.TrustedBundle.MountPath()}, + ) + } + + // Providers without a kube-dns service need a non-default trusted DNS server. + switch rc.Installation.KubernetesProvider { + case operatorv1.ProviderOpenShift: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) + case operatorv1.ProviderRKE2: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) + } + + return env +} + +// mountWindowsPrometheusTLS mounts the node prometheus reporter keypair onto the +// windows daemonset: the volume, the volume mount on each node container, and +// the pod hash annotation that rolls the pods on cert rotation. +func mountWindowsPrometheusTLS(rc extensions.RenderContext, ds *appsv1.DaemonSet) { + tls := windowsData(rc).prometheusServerTLS + if tls == nil { + return + } + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for i := range spec.Containers { + c := &spec.Containers[i] + if windowsNodeContainers[c.Name] { + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeWindows)) + } + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} + +// windowsNodeMetricsService builds the enterprise-only calico-node-metrics-windows +// Service. +func windowsNodeMetricsService(rc extensions.RenderContext) *corev1.Service { + reporterPort := nodeReporterPort(rc.FelixConfiguration) + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.WindowsNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + ClusterIP: "None", + Ports: []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func removeVolumeMount(mounts []corev1.VolumeMount, name string) []corev1.VolumeMount { + out := mounts[:0] + for _, m := range mounts { + if m.Name != name { + out = append(out, m) + } + } + return out +} diff --git a/pkg/enterprise/windows_test.go b/pkg/enterprise/windows_test.go new file mode 100644 index 0000000000..d4c7967b9b --- /dev/null +++ b/pkg/enterprise/windows_test.go @@ -0,0 +1,171 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("windows enterprise image override", func() { + + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + + It("selects the enterprise windows images for the enterprise variant", func() { + Expect(ext.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, ent)).To(Equal(components.ComponentTigeraNodeWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, ent)).To(Equal(components.ComponentTigeraCNIWindows)) + }) + + It("leaves the defaults in place for the Calico variant", func() { + Expect(ext.ResolveImage(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, calico)).To(Equal(components.ComponentCalicoNodeWindows)) + Expect(ext.ResolveImage(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, calico)).To(Equal(components.ComponentCalicoCNIWindows)) + }) +}) + +var _ = Describe("windows enterprise modifier", func() { + + // newObjs returns a windows daemonset with the node containers and the OSS + // cni-log-dir mount the modifier swaps out. + newObjs := func() []client.Object { + nodeContainer := func(name string) corev1.Container { + return corev1.Container{ + Name: name, + VolumeMounts: []corev1.VolumeMount{{MountPath: "/var/log/calico/cni", Name: "cni-log-dir"}}, + } + } + return []client.Object{ + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{nodeContainer("felix"), nodeContainer("node"), nodeContainer("confd")}, + }}}, + }, + } + } + + ds := func(objs []client.Object) *appsv1.DaemonSet { + d, _ := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName) + return d + } + container := func(d *appsv1.DaemonSet, name string) *corev1.Container { + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == name { + return &d.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + ctxFor := func(provider operatorv1.Provider) extensions.RenderContext { + return extensions.RenderContext{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider}, + } + } + + It("appends the node-metrics service", func() { + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) + svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + }) + + It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderNone), newObjs(), nil) + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) + for _, name := range []string{"felix", "node", "confd"} { + c := container(d, name) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", "var-log-calico"))) + Expect(c.VolumeMounts).NotTo(ContainElement(HaveField("Name", "cni-log-dir"))) + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + } + }) + + It("sets the trusted DNS server on openshift", func() { + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctxFor(operatorv1.ProviderOpenShift), newObjs(), nil) + Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) + }) + + It("mounts the prometheus reporter keypair when present", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + cm, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + tls, err := cm.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics-windows"}) + Expect(err).NotTo(HaveOccurred()) + // The installation controller persists the secret; do the same here so the + // windows extension's GetKeyPair finds it. + Expect(cli.Create(context.Background(), tls.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + bundle := cm.CreateTrustedBundle() + + // Build the render context the way the windows controller does: run the + // windows extension, which fetches the keypair into the context. + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: ctxFor(operatorv1.ProviderNone).Installation, + TrustedBundle: bundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: extensions.WindowsController, + Ctx: context.Background(), + Client: cli, + CertificateManager: cm, + } + rc, _, err := ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + + out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, newObjs(), nil) + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) + Expect(d.Spec.Template.Annotations).To(HaveKey(tls.HashAnnotationKey())) + Expect(container(d, "node").Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(container(d, "node").VolumeMounts).To(ContainElement(tls.VolumeMount(render.Windows(&render.WindowsConfiguration{}).SupportedOSType()))) + }) + + It("does nothing for the Calico variant", func() { + ctx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out, _ := applyExtensions(ext, render.ComponentNameWindows, ctx, newObjs(), nil) + _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeFalse()) + Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) + }) +}) diff --git a/pkg/extensions/controllerextension.go b/pkg/extensions/controllerextension.go new file mode 100644 index 0000000000..d61b4cee6f --- /dev/null +++ b/pkg/extensions/controllerextension.go @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// ControllerName identifies the controller a ControllerExtension extends, so a +// variant can register a different hook per controller. Use the constants below +// rather than bare strings so registration and lookup stay in sync. +type ControllerName string + +const ( + InstallationController ControllerName = "installation" + WindowsController ControllerName = "windows" +) + +// ControllerExtension extends a controller's reconcile: it validates the +// configuration and builds the RenderContext the render phase consumes. The core +// operator registers none and runs with the base behavior; an extension build +// registers one per controller it extends. +type ControllerExtension interface { + // Validate rejects configuration the extension does not support, before any + // rendering happens. + Validate(cc ControllerContext) error + + // ExtendContext does the controller-side reconcile work the render phase + // cannot, returning the RenderContext the render phase consumes plus any + // keypairs the extension created that the controller should manage (add to + // certificate management and BYO-expiry warnings), or an error that aborts the + // reconcile. + ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) +} + +// Watcher is an optional companion to ControllerExtension. A controller's Add() +// calls Set.SetupWatches, which invokes Watches on any registered extension that +// implements this, so the extension registers the watches it needs (its CRs, its +// secrets) instead of the controller naming them. +type Watcher interface { + Watches(c ctrlruntime.Controller) error +} + +// ControllerContext is the controller-phase context, the corollary to the +// render-phase RenderContext. It is the embedded RenderContext (the same data +// the render phase sees) plus the controller-side machinery a ControllerExtension +// needs to produce artifacts: a client, a certificate manager, a context. Those +// deps live here, not on RenderContext, so the modifiers that read RenderContext +// can't do I/O - they only transform objects. +// +// Controller names which controller is reconciling, selecting that controller's +// extension hook. The controller fills the embedded RenderContext's data fields, +// the deps, and Controller; ExtendContext does its work, sets the produced +// artifacts on the embedded context, and returns it. +type ControllerContext struct { + RenderContext + + // Controller identifies the reconciling controller, selecting its hook. + Controller ControllerName + + Ctx context.Context + Client client.Client + CertificateManager certificatemanager.CertificateManager +} diff --git a/pkg/extensions/controllerextension_test.go b/pkg/extensions/controllerextension_test.go new file mode 100644 index 0000000000..729e8783d6 --- /dev/null +++ b/pkg/extensions/controllerextension_test.go @@ -0,0 +1,128 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var _ = Describe("controller extension", func() { + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + }) + + It("returns the base render context when the variant has no extension", func() { + install := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + rc, _, err := s.ExtendContext(extensions.ControllerContext{ + RenderContext: extensions.RenderContext{Installation: install, ClusterDomain: "cluster.local"}, + Controller: extensions.InstallationController, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.Installation).To(BeIdenticalTo(install)) + Expect(rc.ClusterDomain).To(Equal("cluster.local")) + Expect(rc.Extension).To(BeNil()) + }) + + It("runs the extension registered for the installation variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{}) + rc, _, err := s.ExtendContext(enterpriseContext()) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("from-fake")) + }) + + It("ignores an extension registered for a different variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{}) + rc, _, err := s.ExtendContext(extensions.ControllerContext{ + RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}, ClusterDomain: "real"}, + Controller: extensions.InstallationController, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + }) + + It("surfaces the extension error", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{err: errors.New("boom")}) + _, _, err := s.ExtendContext(enterpriseContext()) + Expect(err).To(MatchError("boom")) + }) + + It("runs the extension's validation", func() { + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, fakeController{validateErr: errors.New("invalid")}) + Expect(s.Validate(enterpriseContext())).To(MatchError("invalid")) + }) + + It("runs the watch hook of an extension that implements Watcher", func() { + called := false + s.Variant(operatorv1.CalicoEnterprise).Controller(extensions.InstallationController, watchingController{called: &called}) + Expect(s.SetupWatches(extensions.InstallationController, nil)).NotTo(HaveOccurred()) + Expect(called).To(BeTrue()) + }) + + It("returns the base context and no validation error for a nil Set", func() { + var nilSet *extensions.Set + cc := enterpriseContext() + cc.ClusterDomain = "real" + rc, _, err := nilSet.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + Expect(rc.ClusterDomain).To(Equal("real")) + Expect(nilSet.Validate(cc)).NotTo(HaveOccurred()) + }) +}) + +func enterpriseContext() extensions.ControllerContext { + return extensions.ControllerContext{ + RenderContext: extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}}, + Controller: extensions.InstallationController, + } +} + +// fakeController is a ControllerExtension whose Validate and ExtendContext return +// configurable results. +type fakeController struct { + err error + validateErr error +} + +func (f fakeController) Validate(_ extensions.ControllerContext) error { + return f.validateErr +} + +func (f fakeController) ExtendContext(_ extensions.ControllerContext) (extensions.RenderContext, []certificatemanagement.KeyPairInterface, error) { + if f.err != nil { + return extensions.RenderContext{}, nil, f.err + } + return extensions.RenderContext{ClusterDomain: "from-fake"}, nil, nil +} + +// watchingController is a fakeController that also implements the Watcher +// companion, recording that its watch hook ran. +type watchingController struct { + fakeController + called *bool +} + +func (w watchingController) Watches(ctrlruntime.Controller) error { + *w.called = true + return nil +} diff --git a/pkg/extensions/decorate_helpers_test.go b/pkg/extensions/decorate_helpers_test.go new file mode 100644 index 0000000000..82dc31b601 --- /dev/null +++ b/pkg/extensions/decorate_helpers_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// stubExtComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate, the same seam the component +// handler uses. key selects the extension; extCtx is delivered as the component's +// ExtensionContext (the typed config a RegisterModifier modifier reads). +type stubExtComponent struct { + key string + extCtx any + create, delete []client.Object +} + +func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { + return s.create, s.delete +} + +func (s stubExtComponent) Ready() bool { + return true +} + +func (s stubExtComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s stubExtComponent) ModifierKey() string { + return s.key +} + +func (s stubExtComponent) ExtensionContext() any { + return s.extCtx +} + +// applyExtensions decorates a stub component holding the given objects with the +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use applyExtensionsWithContext. +func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return applyExtensionsWithContext(s, key, rc, nil, create, del) +} + +// applyExtensionsWithContext is applyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} + return s.Decorate(stub, rc).Objects() +} diff --git a/pkg/extensions/doc.go b/pkg/extensions/doc.go new file mode 100644 index 0000000000..ec06d9597b --- /dev/null +++ b/pkg/extensions/doc.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extensions is the seam other product variants (today just Calico +// Enterprise) use to layer variant-specific behavior onto the core operator's +// render output, so core code never branches on variant. +// +// A Set holds the extensions for every variant. Per reconcile the controller +// selects one Variant from the installation's variant, so a registered hook only +// ever runs for its own variant and never re-checks it. A Variant bundles two +// kinds of extension: +// +// A ControllerExtension is the controller-side hook. It runs once per reconcile +// in the installation controller, has cluster access (Client, +// CertificateManager) via the ControllerContext, and does the side-effecting +// work a pure render hook can't: rejecting unsupported config (Validate) and +// creating certificates / extending the trusted bundle (ExtendContext). It +// returns the RenderContext, the read-only baton passed to the render phase. +// +// Per-component modifiers are the render phase: pure hooks that run after a +// component builds its objects. An image override swaps the component's image +// (resolved during ResolveImages); a Modifier post-processes the rendered +// objects (run at the componentHandler, which renders the decorated component). +// Register a modifier with Variant.Modify, or with RegisterModifier when it +// needs the component's own typed config. +// +// ControllerContext (controller phase) and RenderContext (render phase) are a +// pair: ControllerContext embeds RenderContext and adds the cluster-access deps, +// which is why modifiers, given only a RenderContext, can't do I/O. +// +// A variant wires up its controller extension and modifiers in one place at +// startup - see pkg/enterprise. +package extensions diff --git a/pkg/extensions/extension.go b/pkg/extensions/extension.go new file mode 100644 index 0000000000..106f1ee8bb --- /dev/null +++ b/pkg/extensions/extension.go @@ -0,0 +1,38 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Modifier post-processes the objects a render component produced. It receives +// the component's create and delete lists and returns the (possibly extended) +// lists. A modifier may mutate matched objects, append objects to create, and +// append objects to delete (e.g. to clean up resources a prior variant left +// behind). It runs only for the variant it is registered under, so it need not +// re-check the variant. +type Modifier func(rc RenderContext, create, delete []client.Object) (newCreate, newDelete []client.Object) + +// FindObject returns the first object of type T with the given name. +func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { + var zero T + for _, o := range objs { + if t, ok := o.(T); ok && o.GetName() == name { + return t, true + } + } + return zero, false +} diff --git a/pkg/extensions/extension_test.go b/pkg/extensions/extension_test.go new file mode 100644 index 0000000000..0ea3359345 --- /dev/null +++ b/pkg/extensions/extension_test.go @@ -0,0 +1,101 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" +) + +var _ = Describe("extension registry", func() { + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + }) + + entCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + + It("applies a registered modifier to the matching component and variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(ctx extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + cm, ok := extensions.FindObject[*corev1.ConfigMap](objs, "cm") + Expect(ok).To(BeTrue()) + cm.Data = map[string]string{"k": "v"} + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del + }) + + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, _ := applyExtensions(s, "test", entCtx, in, nil) + + Expect(out).To(HaveLen(2)) + cm := out[0].(*corev1.ConfigMap) + Expect(cm.Data).To(HaveKeyWithValue("k", "v")) + Expect(out[1].GetName()).To(Equal("extra")) + }) + + It("lets a modifier append to the delete list", func() { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return objs, append(del, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "stale"}}) + }) + + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, del := applyExtensions(s, "test", entCtx, in, nil) + Expect(out).To(Equal(in)) + Expect(del).To(HaveLen(1)) + Expect(del[0].GetName()).To(Equal("stale")) + }) + + It("returns objects unchanged when no modifier is registered", func() { + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, _ := applyExtensions(s, "unregistered", entCtx, in, nil) + Expect(out).To(Equal(in)) + }) + + It("does not apply a modifier registered for a different variant", func() { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "extra"}}), del + }) + + calicoCtx := extensions.RenderContext{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, _ := applyExtensions(s, "test", calicoCtx, in, nil) + Expect(out).To(Equal(in)) + }) + + It("returns objects unchanged when no installation is set", func() { + in := []client.Object{&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}} + out, _ := applyExtensions(s, "test", extensions.RenderContext{}, in, nil) + Expect(out).To(Equal(in)) + }) + + It("replaces rather than stacks when a component modifier is registered twice", func() { + add := func(name string) { + s.Variant(operatorv1.CalicoEnterprise).Modify("test", func(_ extensions.RenderContext, objs, del []client.Object) ([]client.Object, []client.Object) { + return append(objs, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}), del + }) + } + add("first") + add("second") + + out, _ := applyExtensions(s, "test", entCtx, nil, nil) + Expect(out).To(HaveLen(1)) + Expect(out[0].GetName()).To(Equal("second")) + }) +}) diff --git a/pkg/extensions/extensions_suite_test.go b/pkg/extensions/extensions_suite_test.go new file mode 100644 index 0000000000..791eedb737 --- /dev/null +++ b/pkg/extensions/extensions_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestExtensions(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/extensions Suite") +} diff --git a/pkg/extensions/image_test.go b/pkg/extensions/image_test.go new file mode 100644 index 0000000000..736f110ca3 --- /dev/null +++ b/pkg/extensions/image_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" +) + +var _ = Describe("image overrides", func() { + var s *extensions.Set + BeforeEach(func() { + s = extensions.NewSet() + s.Variant(operatorv1.CalicoEnterprise).Image("node", components.ComponentTigeraNode) + }) + + It("uses the override registered for the installation variant", func() { + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + Expect(s.ResolveImage("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + }) + + It("falls back to the default for a variant with no override", func() { + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(s.ResolveImage("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) + }) +}) diff --git a/pkg/extensions/rendercontext.go b/pkg/extensions/rendercontext.go new file mode 100644 index 0000000000..3e092bf18c --- /dev/null +++ b/pkg/extensions/rendercontext.go @@ -0,0 +1,47 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// RenderContext carries reconcile-derived inputs from controllers into render +// modifiers. Core operator code never reads these fields - only registered +// modifiers do. It carries raw cluster state gathered generically (Installation, +// FelixConfiguration, ClusterDomain) that modifiers derive their own values from, +// the shared TrustedBundle, and an opaque Extension slot for controller-produced +// data specific to one extension. +// +// Per-component config a modifier needs but can't derive from these fields is +// not carried here; it flows to the modifier as a typed argument (see +// RegisterModifier), supplied by the component via render.ExtensionContextProvider. +type RenderContext struct { + Installation *operatorv1.InstallationSpec + FelixConfiguration *v3.FelixConfiguration + ClusterDomain string + + // TrustedBundle is the shared CA bundle for the calico-system namespace. + TrustedBundle certificatemanagement.TrustedBundle + + // Extension is opaque, extension-owned data that the controller extension + // produced for its own modifiers - typically an artifact that can only be + // created controller-side because it has cluster side effects (e.g. a keypair). + // The extension that set it type-asserts it back out in its modifiers; core + // code never reads it. Nil when no extension is active. + Extension any +} diff --git a/pkg/extensions/set.go b/pkg/extensions/set.go new file mode 100644 index 0000000000..ee95199a15 --- /dev/null +++ b/pkg/extensions/set.go @@ -0,0 +1,139 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Set is all the variant extensions the operator runs with, indexed by product +// variant. The core operator runs with a nil Set; an extension build (e.g. +// Calico Enterprise) constructs a populated one and hands it in through +// options.ControllerOptions. This replaces what used to be package-level +// registries, so nothing is wired by import side effect. +// +// Per reconcile the controller selects one Variant from the installation's +// variant. The methods the controller calls (Decorate, Validate, ExtendContext, +// Images, ResolveImage) are nil-safe, so the core operator's nil Set yields base +// behavior. +type Set struct { + variants map[operatorv1.ProductVariant]*Variant + images *imageoverride.Overrides +} + +// NewSet returns an empty Set ready to register variant extensions into. +func NewSet() *Set { + return &Set{ + variants: map[operatorv1.ProductVariant]*Variant{}, + images: imageoverride.New(), + } +} + +// Variant returns the extension bundle for v, creating an empty one if needed. +// Used at registration time to build up a variant's extensions. +func (s *Set) Variant(v operatorv1.ProductVariant) *Variant { + if s.variants[v] == nil { + s.variants[v] = &Variant{ + variant: v, + controllers: map[ControllerName]ControllerExtension{}, + modifiers: map[string]decorator{}, + images: s.images, + } + } + return s.variants[v] +} + +// variant looks up the bundle for v, returning nil when none is registered. +// Nil-safe. +func (s *Set) variant(v operatorv1.ProductVariant) *Variant { + if s == nil { + return nil + } + return s.variants[v] +} + +// Decorate wraps component with the extension registered for it under the +// installation's variant, so that when the handler renders the component its +// objects are post-processed by that modifier. A decorated component is itself a +// render.Component, so it flows through the component handler like any other. +// Returns component unchanged when no extension applies. Nil-safe. +func (s *Set) Decorate(component render.Component, ctx RenderContext) render.Component { + if ctx.Installation == nil { + return component + } + return s.variant(ctx.Installation.Variant).decorate(component, ctx) +} + +// Validate runs the cc.Controller extension's validation for the installation's +// variant, or returns nil when no extension is registered. Nil-safe. +func (s *Set) Validate(cc ControllerContext) error { + if cc.Installation == nil { + return nil + } + return s.variant(cc.Installation.Variant).validate(cc) +} + +// ExtendContext runs the cc.Controller extension for the installation's variant +// and returns the resulting RenderContext plus any keypairs the extension wants +// the controller to manage, or the base render context and no keypairs when no +// extension is registered. Nil-safe. +func (s *Set) ExtendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { + if cc.Installation == nil { + return cc.RenderContext, nil, nil + } + return s.variant(cc.Installation.Variant).extendContext(cc) +} + +// SetupWatches registers the watches every variant's extension declares for the +// named controller. It runs at controller startup, which is variant-agnostic, so +// it registers the union across variants (in practice the one active extension +// build's). Nil-safe. +func (s *Set) SetupWatches(controller ControllerName, c ctrlruntime.Controller) error { + if s == nil { + return nil + } + for _, v := range s.variants { + w, ok := v.controllers[controller].(Watcher) + if !ok { + continue + } + if err := w.Watches(c); err != nil { + return err + } + } + return nil +} + +// Images returns the shared image override table. The render package resolves a +// component's image through it directly (the imageoverride leaf, so render need +// not import extensions). Nil-safe, returning nil overrides that resolve to the +// default image. +func (s *Set) Images() *imageoverride.Overrides { + if s == nil { + return nil + } + return s.images +} + +// ResolveImage resolves key for the installation through the image overrides, +// returning def when no override applies. Nil-safe. +func (s *Set) ResolveImage(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + return s.Images().Resolve(key, def, in) +} diff --git a/pkg/extensions/variant.go b/pkg/extensions/variant.go new file mode 100644 index 0000000000..ca1c553c4c --- /dev/null +++ b/pkg/extensions/variant.go @@ -0,0 +1,142 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Variant bundles everything that extends the core operator for one product +// variant: the controller-side hook, the per-component modifiers, and the image +// overrides. The Set selects one Variant per reconcile from the installation's +// variant, so within a Variant there is at most one extension per component and +// nothing here is itself keyed by variant. +type Variant struct { + variant operatorv1.ProductVariant + controllers map[ControllerName]ControllerExtension + modifiers map[string]decorator + images *imageoverride.Overrides // shared with the owning Set +} + +// decorator wraps a base component, returning one whose Objects() are augmented +// by a registered modifier. +type decorator func(base render.Component, rc RenderContext) render.Component + +// Controller registers the variant's controller-side extension for the named +// controller. A controller has at most one; registering replaces any prior one. +func (v *Variant) Controller(name ControllerName, c ControllerExtension) { + v.controllers[name] = c +} + +// Image registers an image override for the named component. +func (v *Variant) Image(component string, image components.Component) { + v.images.Register(v.variant, component, image) +} + +// Modify registers a modifier for a component that needs no per-component +// config. For components whose modifier needs the component's own typed config, +// use RegisterModifier. +func (v *Variant) Modify(component string, m Modifier) { + v.modifiers[component] = func(base render.Component, rc RenderContext) render.Component { + return &decoratedComponent{Component: base, rc: rc, modify: m} + } +} + +// RegisterModifier registers a modifier for component whose modifier needs the +// component's own typed config. The component supplies it via +// render.ExtensionContextProvider; RegisterModifier asserts it to Cfg once, here, +// and hands the typed value to modify - so the modifier body needs no assertion. +// It is a free function because Go has no generic methods. +func RegisterModifier[Cfg any]( + v *Variant, + component string, + modify func(rc RenderContext, cfg Cfg, create, delete []client.Object) ([]client.Object, []client.Object), +) { + v.modifiers[component] = func(base render.Component, rc RenderContext) render.Component { + provider, ok := base.(render.ExtensionContextProvider) + if !ok { + logrus.Errorf("BUG: component %q has a registered modifier but provides no extension context; leaving it unmodified", component) + return base + } + cfg, ok := provider.ExtensionContext().(Cfg) + if !ok { + var want Cfg + logrus.Errorf("BUG: component %q extension context is %T, want %T; leaving it unmodified", component, provider.ExtensionContext(), want) + return base + } + bound := func(rc RenderContext, create, delete []client.Object) ([]client.Object, []client.Object) { + return modify(rc, cfg, create, delete) + } + return &decoratedComponent{Component: base, rc: rc, modify: bound} + } +} + +// decorate wraps component with the modifier registered for its extension key, +// or returns it unchanged when the component exposes no extension point or none +// is registered. Nil-safe. +func (v *Variant) decorate(component render.Component, rc RenderContext) render.Component { + if v == nil { + return component + } + ext, ok := component.(render.Extensible) + if !ok { + return component + } + build, ok := v.modifiers[ext.ModifierKey()] + if !ok { + return component + } + return build(component, rc) +} + +// validate runs the cc.Controller extension's validation, or nil when the +// variant has none for it. Nil-safe. +func (v *Variant) validate(cc ControllerContext) error { + if v == nil || v.controllers[cc.Controller] == nil { + return nil + } + return v.controllers[cc.Controller].Validate(cc) +} + +// extendContext runs the cc.Controller extension, or returns the base render +// context and no managed keypairs when the variant has none for it. Nil-safe. +func (v *Variant) extendContext(cc ControllerContext) (RenderContext, []certificatemanagement.KeyPairInterface, error) { + if v == nil || v.controllers[cc.Controller] == nil { + return cc.RenderContext, nil, nil + } + return v.controllers[cc.Controller].ExtendContext(cc) +} + +// decoratedComponent is the render.Component produced by decorate: it renders +// its embedded base component and then runs the variant modifier over the +// result. It embeds the base render.Component, so ResolveImages, SupportedOSType, +// and Ready delegate to the base; only Objects is augmented. +type decoratedComponent struct { + render.Component + rc RenderContext + modify Modifier +} + +func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { + create, del := d.Component.Objects() + return d.modify(d.rc, create, del) +} diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go new file mode 100644 index 0000000000..ee4c48d559 --- /dev/null +++ b/pkg/imageoverride/imageoverride.go @@ -0,0 +1,61 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package imageoverride is a leaf package (no render/operator dependencies) +// that holds the image override table. The render package imports it to resolve +// a component's image without depending on pkg/extensions, which would cycle. +package imageoverride + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" +) + +type overrideKey struct { + variant operatorv1.ProductVariant + key string +} + +// Overrides maps a component (keyed by variant) to the image it should resolve +// to, letting a variant swap a component's image without the render package +// branching on variant. The render component holds one and resolves through it. +// Registry, image path, and FIPS handling are applied downstream in the render +// package, so an override only picks which component. +type Overrides struct { + m map[overrideKey]components.Component +} + +// New returns an empty Overrides. +func New() *Overrides { + return &Overrides{m: map[overrideKey]components.Component{}} +} + +// Register stores image under key for the given variant. The key is the render +// component's image identifier (e.g. "node"). +func (o *Overrides) Register(variant operatorv1.ProductVariant, key string, image components.Component) { + o.m[overrideKey{variant, key}] = image +} + +// Resolve returns the override registered for key under the installation's +// variant, otherwise def. It is safe to call on a nil *Overrides (the core +// operator hands render no overrides), which always returns def. +func (o *Overrides) Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + if o == nil || in == nil { + return def + } + if image, ok := o.m[overrideKey{in.Variant, key}]; ok { + return image + } + return def +} diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go index 311d5b05a3..9702226fd3 100644 --- a/pkg/render/apiserver.go +++ b/pkg/render/apiserver.go @@ -40,7 +40,6 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/render/common/authentication" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/podaffinity" @@ -57,8 +56,9 @@ const ( APIServerPortName = "apiserver" APIServerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "apiserver-access" - auditLogsVolumeName = "calico-audit-logs" - auditPolicyVolumeName = "calico-audit-policy" + // ComponentNameAPIServer is the extension key under which a variant registers + // its API server modifier and image override. + ComponentNameAPIServer = "apiserver" ) const ( @@ -147,6 +147,12 @@ type APIServerConfiguration struct { // as part of this component. RequiresAggregationServer bool + // Whether or not the API server deployment must run a query server alongside the API + // server. The deployment (and its supporting objects) are rendered when either an + // aggregation API server or a query server is required. The query server itself, and + // the rest of its supporting configuration, is layered on by the variant's modifier. + RequiresQueryServer bool + // When certificate management is enabled, we need a separate init container to create a cert, running // with the same permissions as query server. QueryServerTLSKeyPairCertificateManagementOnly certificatemanagement.KeyPairInterface @@ -159,6 +165,25 @@ type apiServerComponent struct { dikastesImage string } +// APIServerExtensionContext carries the API server's render configuration and resolved +// image to a variant modifier. The modifier uses these to build variant-specific objects +// and to layer additional containers, volumes, and configuration onto the rendered +// deployment. +type APIServerExtensionContext struct { + Config *APIServerConfiguration + CalicoImage string +} + +// ModifierKey implements render.Extensible: the API server's variant-specific objects are +// applied by the modifier registered under this key. +func (c *apiServerComponent) ModifierKey() string { return ComponentNameAPIServer } + +// ExtensionContext implements render.ExtensionContextProvider, handing the modifier the +// config and resolved image it needs. +func (c *apiServerComponent) ExtensionContext() any { + return APIServerExtensionContext{Config: c.cfg, CalicoImage: c.calicoImage} +} + func (c *apiServerComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := c.cfg.Installation.Registry path := c.cfg.Installation.ImagePath @@ -166,15 +191,14 @@ func (c *apiServerComponent) ResolveImages(is *operatorv1.ImageSet) error { var err error errMsgs := []string{} - enterprise := c.cfg.Installation.Variant.IsEnterprise() - if enterprise || c.cfg.RequiresAggregationServer { + if c.cfg.RequiresAggregationServer || c.cfg.RequiresQueryServer { c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) if err != nil { errMsgs = append(errMsgs, err.Error()) } } - if enterprise && c.cfg.IsSidecarInjectionEnabled() { + if c.cfg.IsSidecarInjectionEnabled() { c.l7AdmissionControllerEnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, reg, path, prefix, is) if err != nil { errMsgs = append(errMsgs, err.Error()) @@ -196,8 +220,8 @@ func (c *apiServerComponent) SupportedOSType() rmeta.OSType { } func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { - // Start with all of the cluster-scoped resources that are used for both Calico and Calico Enterprise. - // When switching between Calico / Enterprise, these objects are simply updated in-place. + // Cluster-scoped resources used by the API server, independent of variant. Any + // variant-specific objects are layered on by the variant's modifier. globalObjects := []client.Object{ c.calicoCustomResourcesClusterRole(), c.calicoCustomResourcesClusterRoleBinding(), @@ -209,9 +233,6 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { } objsToDelete := []client.Object{} - - // Namespaced objects common to both Calico and Calico Enterprise. - // These objects will be updated when switching between the variants. namespacedObjects := []client.Object{} // Add in image pull secrets. @@ -219,8 +240,8 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { namespacedObjects = append(namespacedObjects, secret.ToRuntimeObjects(secrets...)...) // The deployment and its supporting objects are needed when running the aggregation API server - // or when running Enterprise (which always needs the queryserver). - if c.cfg.RequiresAggregationServer || c.cfg.Installation.Variant.IsEnterprise() { + // or when a query server runs alongside it (the query server is added by a variant modifier). + if c.cfg.RequiresAggregationServer || c.cfg.RequiresQueryServer { namespacedObjects = append(namespacedObjects, c.apiServerServiceAccount(), c.apiServerDeployment(), @@ -246,16 +267,6 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { c.authReaderRoleBinding(), } - if c.cfg.Installation.Variant.IsEnterprise() { - aggregationAPIServerObjects = append(aggregationAPIServerObjects, - c.uiSettingsGroupGetterClusterRole(), - c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), - c.uiSettingsPassthruClusterRole(), - c.uiSettingsPassthruClusterRolebinding(), - c.auditPolicyConfigMap(), - ) - } - // Add in certificates for API server TLS. if !c.cfg.TLSKeyPair.UseCertificateManagement() { aggregationAPIServerObjects = append(aggregationAPIServerObjects, c.apiServiceRegistration(c.cfg.TLSKeyPair.GetCertificatePEM())) @@ -263,73 +274,12 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { aggregationAPIServerObjects = append(aggregationAPIServerObjects, c.apiServiceRegistration(c.cfg.Installation.CertificateManagement.CACert)) } - // Global enterprise-only objects. - globalEnterpriseObjects := []client.Object{ - c.tigeraAPIServerClusterRole(), - c.tigeraAPIServerClusterRoleBinding(), - } - - if !c.cfg.MultiTenant { - // These resources are only installed in zero-tenant clusters. Multi-tenant clusters don't use the default - // RBAC resources. - globalEnterpriseObjects = append(globalEnterpriseObjects, - c.tigeraUserClusterRole(), - c.tigeraNetworkAdminClusterRole(), - ) - } - - if c.cfg.ManagementCluster != nil { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.managedClusterWatchClusterRole()) - if c.cfg.MultiTenant { - // Multi-tenant management cluster API servers need access to per-tenant CA secrets in order to sign - // per-tenant guardian certificates when creating ManagedClusters. - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantSecretsRBAC()...) - // Multi-tenant management cluster components impersonate the single-tenant canonical service account - // in order to retrieve informations from the managed cluster. A cluster role will be created and each - // component will create a role binding in the tenant namespace - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantManagedClusterAccessClusterRoles()...) - } else { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.secretsRBAC()...) - } - } else { - // If we're not a management cluster, the API server doesn't need permissions to access secrets. - objsToDelete = append(objsToDelete, c.multiTenantSecretsRBAC()...) - objsToDelete = append(objsToDelete, c.secretsRBAC()...) - objsToDelete = append(objsToDelete, c.multiTenantManagedClusterAccessClusterRoles()...) - objsToDelete = append(objsToDelete, c.managedClusterWatchClusterRole()) - } - - // Namespaced enterprise-only objects. - namespacedEnterpriseObjects := []client.Object{} - - if c.cfg.TrustedBundle != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.cfg.TrustedBundle.ConfigMap(QueryserverNamespace)) - } + // The sidecar mutating webhook is driven by ApplicationLayer configuration, not by variant. if c.cfg.IsSidecarInjectionEnabled() { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.sidecarMutatingWebhookConfig()) + namespacedObjects = append(namespacedObjects, c.sidecarMutatingWebhookConfig()) } else { objsToDelete = append(objsToDelete, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) } - if c.cfg.ManagementClusterConnection != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, - c.externalLinseedRoleBinding(), - ) - } - - // Compile the final arrays based on the variant. - if c.cfg.Installation.Variant.IsEnterprise() { - // Create any enterprise specific objects. - globalObjects = append(globalObjects, globalEnterpriseObjects...) - namespacedObjects = append(namespacedObjects, namespacedEnterpriseObjects...) - - // Clean up cluster-scoped resources that were created with the 'tigera' prefix. - // The apiserver now uses consistent resource names with 'calico' prefix across both EE and OSS variants. - objsToDelete = append(objsToDelete, c.deprecatedResources()...) - } else { - // Explicitly delete any global enterprise objects. - // Namespaced objects will be handled by namespace deletion. - objsToDelete = append(objsToDelete, globalEnterpriseObjects...) - } // Clean up deprecated k8s NetworkPolicy, regardless of variant, // avoiding leftovers in the case of switching between variants. @@ -523,9 +473,9 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy Action: v3.Pass, }) - apiServerContainerPort := getContainerPort(cfg, APIServerContainerName).ContainerPort - queryServerContainerPort := getContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - l7AdmCtrlContainerPort := getContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort + apiServerContainerPort := GetContainerPort(cfg, APIServerContainerName).ContainerPort + queryServerContainerPort := GetContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort + l7AdmCtrlContainerPort := GetContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort // The ports Calico Enterprise API Server and Calico Enterprise Query Server are configured to listen on. ingressPorts := networkpolicy.Ports(443, uint16(apiServerContainerPort), uint16(queryServerContainerPort), 10443) @@ -768,17 +718,6 @@ func (c *apiServerComponent) authClusterRole() client.Object { } } -// multiTenantSecretsRBAC provides the tigera API server with the ability to read secrets on the cluster. -// This is needed in multi-tenant management clusters only, in order to read tenant secrets for signing managed cluster certificates. -func (c *apiServerComponent) multiTenantSecretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, true) -} - -// secretsRBAC provides the tigera API server with the ability to read secrets from the API server's namespace. -func (c *apiServerComponent) secretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, false) -} - // authClusterRoleBinding returns a clusterrolebinding to create, and a clusterrolebinding to delete. // // Both Calico and Calico Enterprise, with different names. @@ -854,7 +793,7 @@ func (c *apiServerComponent) webhookReaderClusterRoleBinding() client.Object { } } -func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) *operatorv1.APIServerDeploymentContainerPort { +func GetContainerPort(cfg *APIServerConfiguration, containerName ContainerName) *operatorv1.APIServerDeploymentContainerPort { // Try to get the override port if cfg != nil && cfg.APIServer != nil && @@ -889,11 +828,11 @@ func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) return nil } -// apiServerService creates a service backed by the API server and - for enterprise - query server. +// apiServerService creates a service backed by the API server. A variant modifier may add +// additional ports (e.g. the query server port). func (c *apiServerComponent) apiServerService() *corev1.Service { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName) - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName) - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName) + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName) + l7AdmissionControllerTargetPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName) s := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, @@ -917,18 +856,6 @@ func (c *apiServerComponent) apiServerService() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add port for queryserver if enterprise. - s.Spec.Ports = append(s.Spec.Ports, - corev1.ServicePort{ - Name: QueryServerPortName, - Port: QueryServerPort, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), - }, - ) - } - if c.cfg.IsSidecarInjectionEnabled() { s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ @@ -967,12 +894,13 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { initContainers = append(initContainers, initContainerAPIServer) } - initContainerQueryServer := c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.InitContainer(APIServerNamespace, c.queryServerContainer().SecurityContext) + initContainerQueryServer := c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.InitContainer(APIServerNamespace, securitycontext.NewNonRootContext()) annotations[c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationKey()] = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationValue() initContainers = append(initContainers, initContainerQueryServer) } - // Determine which containers to run. + // Determine which containers to run. A variant modifier may add additional + // containers (e.g. the query server). containers := []corev1.Container{} if c.cfg.RequiresAggregationServer { containers = append(containers, c.apiServerContainer()) @@ -980,9 +908,6 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { if c.cfg.IsSidecarInjectionEnabled() { containers = append(containers, c.l7AdmissionControllerContainer()) } - if c.cfg.Installation.Variant.IsEnterprise() { - containers = append(containers, c.queryServerContainer()) - } d := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, @@ -1027,15 +952,6 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { d.Spec.Template.Spec.Affinity = podaffinity.NewPodAntiAffinity(APIServerName, []string{APIServerNamespace, "tigera-system", "calico-apiserver"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.TrustedBundle != nil { - trustedBundleHashAnnotations := c.cfg.TrustedBundle.HashAnnotations() - for k, v := range trustedBundleHashAnnotations { - d.Spec.Template.Annotations[k] = v - } - } - } - if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { rcomp.ApplyDeploymentOverrides(d, overrides) } @@ -1046,7 +962,7 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { // apiServer creates a MutatingWebhookConfiguration for sidecars. func (c *apiServerComponent) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConfiguration { var cacert []byte - svcPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort + svcPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort svcpath := "/sidecar-webhook" svcref := admregv1.ServiceReference{ @@ -1103,10 +1019,16 @@ func (c *apiServerComponent) sidecarMutatingWebhookConfig() *admregv1.MutatingWe } func (c *apiServerComponent) hostNetwork() bool { - if c.cfg.ForceHostNetwork { + return HostNetwork(c.cfg) +} + +// HostNetwork reports whether the API server deployment runs on the host network, +// accounting for both the forced setting and the provider-driven requirement. +func HostNetwork(cfg *APIServerConfiguration) bool { + if cfg.ForceHostNetwork { return true } - return HostNetworkRequired(c.cfg.Installation) + return HostNetworkRequired(cfg.Installation) } func HostNetworkRequired(installation *operatorv1.InstallationSpec) bool { @@ -1126,12 +1048,6 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { volumeMounts := []corev1.VolumeMount{ c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), } - if c.cfg.Installation.Variant.IsEnterprise() { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, - corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, - ) - } env := []corev1.EnvVar{ {Name: "DATASTORE_TYPE", Value: "kubernetes"}, @@ -1161,7 +1077,7 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) } - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort apiServer := corev1.Container{ Name: string(APIServerContainerName), @@ -1183,19 +1099,13 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { PeriodSeconds: 60, }, } - // In case of OpenShift, apiserver needs privileged access to write audit logs to host path volume. - // Audit logs are owned by root on hosts so we need to be root user and group. Audit logs are supported only in Enterprise version. - if c.cfg.Installation.Variant.IsEnterprise() { - apiServer.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) - } else { - apiServer.SecurityContext = securitycontext.NewNonRootContext() - } + apiServer.SecurityContext = securitycontext.NewNonRootContext() return apiServer } func (c *apiServerComponent) startUpArgs() []string { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort args := []string{ fmt.Sprintf("--secure-port=%d", apiServerTargetPort), @@ -1203,13 +1113,6 @@ func (c *apiServerComponent) startUpArgs() []string { fmt.Sprintf("--tls-cert-file=%s", c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()), } - if c.cfg.Installation.Variant.IsEnterprise() { - args = append(args, - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - ) - } - if c.cfg.ManagementCluster != nil { args = append(args, "--enable-managed-clusters-create-api=true") if c.cfg.ManagementCluster.Spec.Address != "" { @@ -1230,122 +1133,6 @@ func (c *apiServerComponent) startUpArgs() []string { return args } -// queryServerContainer creates the query server container. -func (c *apiServerComponent) queryServerContainer() corev1.Container { - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - - var tlsSecret certificatemanagement.KeyPairInterface - if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { - tlsSecret = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly - } else { - tlsSecret = c.cfg.TLSKeyPair - } - env := []corev1.EnvVar{ - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, - {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) - } - - if c.hostNetwork() { - env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) - } else { - env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.KeyValidatorConfig != nil { - env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) - } - - linseedURL := relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, ElasticsearchNamespace, c.cfg.ManagementClusterConnection != nil, false) - env = append(env, - corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, - corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - ) - if c.cfg.ManagementClusterConnection != nil { - env = append(env, - corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, - corev1.EnvVar{Name: "LINSEED_TOKEN", Value: GetLinseedTokenPath(true)}, - ) - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) - } - - // set LogLEVEL for queryserver container - if logging := c.cfg.APIServer.Logging; logging != nil && - logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { - env = append(env, - corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) - } else { - // set default LOGLEVEL to info when not set by the user - env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) - } - - volumeMounts := []corev1.VolumeMount{ - tlsSecret.VolumeMount(c.SupportedOSType()), - } - if c.cfg.TrustedBundle != nil { - volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) - } - if c.cfg.ManagementClusterConnection != nil { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: LinseedTokenVolumeName, - MountPath: LinseedVolumeMountPath, - }) - } - - container := corev1.Container{ - Name: string(TigeraAPIServerQueryServerContainerName), - Image: c.calicoImage, - Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, - Env: env, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/version", - Port: intstr.FromInt32(queryServerTargetPort), - Scheme: corev1.URISchemeHTTPS, - }, - }, - InitialDelaySeconds: 90, - }, - SecurityContext: securitycontext.NewNonRootContext(), - VolumeMounts: volumeMounts, - } - return container -} - -func (c *apiServerComponent) externalLinseedRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-linseed", - Namespace: APIServerNamespace, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: TigeraLinseedSecretsClusterRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - // apiServerVolumes creates the volumes used by the API server deployment. func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { volumes := []corev1.Volume{ @@ -1355,40 +1142,6 @@ func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { volumes = append(volumes, c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.Volume()) } - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.RequiresAggregationServer { - // Only include these volumes if we're running the aggregation API server, since audit logging is done through the - // main API server otherwise. - volumes = append(volumes, - corev1.Volume{ - Name: auditLogsVolumeName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/log/calico/audit", - Type: ptr.To(corev1.HostPathDirectoryOrCreate), - }, - }, - }, - corev1.Volume{ - Name: auditPolicyVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, - Items: []corev1.KeyToPath{ - { - Key: "config", - Path: "policy.conf", - }, - }, - }, - }, - }, - ) - } - - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.TrustedBundle != nil { - volumes = append(volumes, c.cfg.TrustedBundle.Volume()) - } - if c.cfg.ManagementClusterConnection != nil { // Optional: the Secret is delivered over the Guardian tunnel, which can't be // established until calico-apiserver is Ready. @@ -1419,119 +1172,6 @@ func (c *apiServerComponent) tolerations() []corev1.Toleration { return tolerations } -// tigeraAPIServerClusterRole creates a clusterrole that gives permissions to access backing CRDs -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - { - // Read access to Linseed policy activity data for queryserver enrichment. - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{"policyactivity"}, - Verbs: []string{"get"}, - }, - { - // Calico Enterprise backing storage. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "bfdconfigurations", - "deeppacketinspections", - "deeppacketinspections/status", - "egressgatewaypolicies", - "externalnetworks", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalreports", - "globalreports/status", - "globalreporttypes", - "globalthreatfeeds", - "globalthreatfeeds/status", - "licensekeys", - "managedclusters", - "managedclusters/status", - "networks", - "packetcaptures", - "packetcaptures/status", - "policyrecommendationscopes", - "policyrecommendationscopes/status", - "remoteclusterconfigurations", - "securityeventwebhooks", - "securityeventwebhooks/status", - "uisettings", - "uisettingsgroups", - }, - Verbs: []string{ - "get", - "list", - "watch", - "create", - "update", - "delete", - "patch", - }, - }, - { - // The queryserver's RBAC calculator needs to list tiers, - // uisettingsgroups, and managedclusters via the aggregated - // API to evaluate user permissions for the /policies endpoint. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "tiers", - "uisettingsgroups", - "managedclusters", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Required by the AuthorizationReview calculator in queryserver to evaluate - // RBAC permissions for users. - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{ - "clusterroles", - "clusterrolebindings", - "roles", - "rolebindings", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Rules: rules, - } -} - -// tigeraAPIServerClusterRoleBinding creates a clusterrolebinding that applies tigeraAPIServerClusterRole to -// the calico-apiserver service account. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: APIServerServiceAccountName, - Namespace: APIServerNamespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: APIServerName, - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - // tierGetterClusterRole creates a clusterrole that gives permissions to get tiers. func (c *apiServerComponent) tierGetterClusterRole() *rbacv1.ClusterRole { return &rbacv1.ClusterRole{ @@ -1575,711 +1215,86 @@ func (c *apiServerComponent) kubeControllerMgrTierGetterClusterRoleBinding() *rb } } -// uiSettingsGroupGetterClusterRole creates a clusterrole that gives permissions to get uisettingsgroups. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { +// calicoPolicyPassthruClusterRole creates a clusterrole that is used to control the RBAC +// mechanism for Calico tiered policy. +func (c *apiServerComponent) calicoPolicyPassthruClusterRole() *rbacv1.ClusterRole { + resources := []string{"networkpolicies", "globalnetworkpolicies"} + return &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", + Name: "calico-tiered-policy-passthrough", }, + // If tiered policy is enabled we allow all authenticated users to access the main tier resource, instead + // restricting access using the tier.xxx resource type. Kubernetes NetworkPolicy and + // StagedKubernetesNetworkPolicy objects are handled using normal (non-tiered) RBAC. Rules: []rbacv1.PolicyRule{ { APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "uisettingsgroups", - }, - Verbs: []string{"get"}, + Resources: resources, + Verbs: allVerbs, }, }, } } -// kubeControllerManagerUISettingsGroupGetterClusterRoleBinding creates a rolebinding that allows the k8s kube-controller -// manager to get uisettingsgroups. -// -// In k8s 1.15+, cascading resource deletions (for instance pods for a replicaset) failed due to k8s kube-controller -// not having permissions to get tiers. UISettings and UISettingsGroups RBAC works in a similar way to tiered policy -// and so we need similar RBAC for UISettingsGroups. -// -// Calico Enterprise only -func (c *apiServerComponent) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { +// calicoPolicyPassthruClusterRolebinding creates a clusterrolebinding that applies calicoPolicyPassthruClusterRole to all users. +func (c *apiServerComponent) calicoPolicyPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettingsgroup-getter", - APIGroup: "rbac.authorization.k8s.io", + Name: "calico-tiered-policy-passthrough", }, Subjects: []rbacv1.Subject{ { - Kind: "User", - Name: "system:kube-controller-manager", + Kind: "Group", + Name: "system:authenticated", APIGroup: "rbac.authorization.k8s.io", }, }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-tiered-policy-passthrough", + APIGroup: "rbac.authorization.k8s.io", + }, } } -// tigeraUserClusterRole returns a cluster role for a default Calico Enterprise user. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraUserClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // List requests that the Tigera manager needs. - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - "", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "namespaces", - "globalnetworksets", - "networksets", - "managedclusters", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "policyrecommendationscopes", - }, - Verbs: []string{"watch", "list"}, +func (c *apiServerComponent) getDeprecatedResources() []client.Object { + var renamedRscList []client.Object + + // renamed clusterrole tigera-crds to tigera-apiserver + renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-crds", }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to view Networks. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Access to policies in all tiers - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"get"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // List and view the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"get", "watch", "list"}, - }, - // User can: - // - read UISettings in the cluster-settings group - // - read and write UISettings in the user-settings group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch"}, - ResourceNames: []string{"cluster-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"user-settings"}, - }, - // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read services to view WAF configuration. - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to only view securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list"}, - }, - } - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Access to log into Kibana for oidc users. - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", - }, - Verbs: []string{"get"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-ui-user", - }, - Rules: rules, - } -} - -// tigeraNetworkAdminClusterRole returns a cluster role for a Tigera Secure manager network admin. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // Full access to all network policies - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "globalnetworksets", - "networksets", - "managedclusters", - "packetcaptures", - "policyrecommendationscopes", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{ - "policy.networking.k8s.io", - }, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get", "delete"}, - }, - // Allow the user to CRUD Networks. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - // Additional "list" requests that the Tigera Secure manager needs - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"watch", "list"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports/status"}, - Verbs: []string{"get", "list", "watch"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - // Access to cluster information containing Calico and EE versions from the UI. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // Manage the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - // User can: - // - read and write UISettings in the cluster-settings group, and rename the group - // - read and write UISettings in the user-settings group, and rename the group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get", "patch", "update"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - // Allow the user to read and write applicationlayers to enable/disable WAF. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get", "update", "patch", "create", "delete"}, - }, - // Allow the user to read deployments to view WAF configuration. - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to perform CRUD operations on securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, - }, - // Allow the user to create secrets. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - Verbs: []string{"create"}, - }, - // Allow the user to patch webhooks-secret secret. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - ResourceNames: []string{ - "webhooks-secret", - }, - Verbs: []string{"patch"}, - }, - } - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Elasticsearch superuser access once logged into Kibana. - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", - }, - Verbs: []string{"get"}, - }) - } - - // In v3 CRD / webhooks mode there is no aggregated apiserver, and the - // calico-uisettings-passthrough ClusterRole that normally grants the broad - // uisettings permission isn't deployed. Grant write verbs here so the - // calico-webhooks UISettings handler (which narrows access via a SAR on - // uisettingsgroups/data) gets invoked instead of being short-circuited by - // kube-apiserver RBAC. - if !c.cfg.RequiresAggregationServer { - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"create", "update", "delete", "patch"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-network-admin", - }, - Rules: rules, - } -} - -// calicoPolicyPassthruClusterRole creates a clusterrole that is used to control the RBAC -// mechanism for Calico tiered policy. -func (c *apiServerComponent) calicoPolicyPassthruClusterRole() *rbacv1.ClusterRole { - resources := []string{"networkpolicies", "globalnetworkpolicies"} - - // Append additional resources for enterprise Variant. - if c.cfg.Installation.Variant.IsEnterprise() { - resources = append(resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tiered-policy-passthrough", - }, - // If tiered policy is enabled we allow all authenticated users to access the main tier resource, instead - // restricting access using the tier.xxx resource type. Kubernetes NetworkPolicy and - // StagedKubernetesNetworkPolicy objects are handled using normal (non-tiered) RBAC. - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: resources, - Verbs: allVerbs, - }, - }, - } -} + }) -// calicoPolicyPassthruClusterRolebinding creates a clusterrolebinding that applies calicoPolicyPassthruClusterRole to all users. -func (c *apiServerComponent) calicoPolicyPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ + // renamed clusterrolebinding tigera-apiserver-access-tigera-crds to tigera-apiserver + renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tiered-policy-passthrough", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:authenticated", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-tiered-policy-passthrough", - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// uiSettingsPassthruClusterRole creates a clusterrole that is used to control the RBAC mechanism for Tigera UI Settings. -// RBAC for these is handled within the Tigera API Server which checks uisettingsgroups/data permissions for the user. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"*"}, - }, + Name: "tigera-apiserver-access-tigera-crds", }, - } -} + }) -// uiSettingsPassthruClusterRolebinding creates a clusterrolebinding that applies uiSettingsPassthruClusterRole to all -// users. -// -// Calico Enterprise only. -func (c *apiServerComponent) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ + // Renamed ClusterRoleBinding tigera-tier-getter to calico-tier-getter since Tier is available in OSS. + // Deleting an object that was never created (e.g. in a fresh OSS install) is a no-op. + renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:authenticated", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettings-passthrough", - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// auditPolicyConfigMap returns a configmap with contents to configure audit logging for -// projectcalico.org/v3 APIs. -// -// Calico Enterprise only -func (c *apiServerComponent) auditPolicyConfigMap() *corev1.ConfigMap { - const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 -kind: Policy -rules: -- level: RequestResponse - omitStages: - - RequestReceived - verbs: - - create - - patch - - update - - delete - resources: - - group: projectcalico.org - resources: - - globalnetworkpolicies - - networkpolicies - - stagedglobalnetworkpolicies - - stagednetworkpolicies - - stagedkubernetesnetworkpolicies - - globalnetworksets - - networksets - - tiers - - hostendpoints` - - return &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - // This object is for Enterprise only, so pass it explicitly. - Namespace: APIServerNamespace, - Name: auditPolicyVolumeName, - }, - Data: map[string]string{ - "config": defaultAuditPolicy, - }, - } -} - -func (c *apiServerComponent) multiTenantManagedClusterAccessClusterRoles() []client.Object { - var objects []client.Object - objects = append(objects, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: MultiTenantManagedClustersAccessClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for - // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. - // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. - "get", - }, - }, + Name: "tigera-tier-getter", }, }) - - return objects -} - -// managedClusterWatchClusterRole creates a ClusterRole for watching the ManagedCluster API -func (c *apiServerComponent) managedClusterWatchClusterRole() client.Object { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: ManagedClustersWatchClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - "get", "list", "watch", - }, - }, - }, - } -} - -func (c *apiServerComponent) getDeprecatedResources() []client.Object { - var renamedRscList []client.Object - - // renamed clusterrole tigera-crds to tigera-apiserver + // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-crds", - }, - }) - - // renamed clusterrolebinding tigera-apiserver-access-tigera-crds to tigera-apiserver - renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-apiserver-access-tigera-crds", + Name: "tigera-tier-getter", }, }) - // The following resources were not present in Calico OSS, so there is no need to clean up in OSS. - if c.cfg.Installation.Variant.IsEnterprise() { - // Renamed ClusterRoleBinging tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - } - renamedRscList = append(renamedRscList, &corev1.Namespace{ TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -2314,7 +1329,7 @@ func (c *apiServerComponent) l7AdmissionControllerContainer() corev1.Container { c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), } - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort + l7AdmissionControllerTargetPort := GetContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort dataplane := "iptables" if c.cfg.Installation.IsNftables() { @@ -2365,99 +1380,3 @@ func (c *apiServerComponent) l7AdmissionControllerContainer() corev1.Container { return l7AdmssCtrl } - -// deprecatedResources removes legacy cluster-scoped resources created with the 'tigera' prefix (EE-only). -// Moving forward, both EE and OSS variants standardize on the 'calico' prefix for all shared resources. -// TODO to clean up the below deprecated logic with 14 resources in 3.25+ -func (c *apiServerComponent) deprecatedResources() []client.Object { - return []client.Object{ - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - - // delegateAuthClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, - }, - - // authClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - - // authClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - // authReaderRoleBinding - need clean up in diff namespace kube-system - &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-auth-reader", - Namespace: "kube-system", - }, - }, - // webhookReaderClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, - }, - - // webhookReaderClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, - }, - - // calico-apiserver CR and CRB - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - - // Clean up legacy secrets in the tigera-operator namespace - &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, - }, - } -} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index daecd8593d..57eeb83e66 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -38,6 +38,7 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -60,6 +61,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// apiServerObjects renders the API server component and applies the registered variant +// modifier the way the componentHandler does, so the Enterprise objects (query server, +// audit logging, Enterprise RBAC) and the Calico cleanup deletes are reflected in the +// returned create and delete lists. +func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { + create, del := c.Objects() + rc := extensions.RenderContext{} + var extCtx any + if p, ok := c.(render.ExtensionContextProvider); ok { + ec := p.ExtensionContext().(render.APIServerExtensionContext) + rc.Installation = ec.Config.Installation + extCtx = ec + } + return applyExtensionsWithContext(ext, render.ComponentNameAPIServer, rc, extCtx, create, del) +} + var _ = Describe("API server rendering tests (Calico Enterprise)", func() { apiServerPolicy := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver.json") apiServerPolicyForOCP := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver_ocp.json") @@ -97,6 +114,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg = &render.APIServerConfiguration{ RequiresAggregationServer: true, + RequiresQueryServer: true, K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, Installation: instance, APIServer: apiserver, @@ -149,7 +167,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) // Should render the correct resources. // - 1 namespace @@ -389,7 +407,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expectedResources := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, @@ -427,7 +445,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { It("should grant the calico-apiserver SA write access to globalreports/status", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) cr := rtest.GetResource(resources, "calico-apiserver", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -465,7 +483,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -503,7 +521,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { } component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -535,7 +553,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred()) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) role := rtest.GetResource(resources, "calico-extension-apiserver-auth-access", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ @@ -585,7 +603,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -630,7 +648,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -699,7 +717,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -720,7 +738,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) }) @@ -759,7 +777,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) @@ -790,7 +808,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -803,7 +821,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.ForceHostNetwork = true component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) }) @@ -814,7 +832,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.ForceHostNetwork = true component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) Expect(policy).ToNot(BeNil()) @@ -836,7 +854,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.ForceHostNetwork = false component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) Expect(policy).ToNot(BeNil()) @@ -860,7 +878,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -877,7 +895,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -892,7 +910,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expectedResources := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, @@ -958,7 +976,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expected := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}}, @@ -1018,7 +1036,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(dep).ToNot(BeNil()) @@ -1035,7 +1053,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(dep).ToNot(BeNil()) @@ -1057,7 +1075,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { Expect(err).NotTo(HaveOccurred()) component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) expectedResources := []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, @@ -1102,7 +1120,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.Installation.ControlPlaneReplicas = ptr.To(int32(1)) component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -1113,7 +1131,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { cfg.Installation.ControlPlaneReplicas = ptr.To(int32(2)) component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -1127,7 +1145,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) rb, ok := rtest.GetResource(resources, "tigera-linseed", "calico-system", "rbac.authorization.k8s.io", "v1", "RoleBinding").(*rbacv1.RoleBinding) Expect(ok).To(BeTrue(), "expected tigera-linseed RoleBinding in calico-system") @@ -1179,7 +1197,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { } component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) expectedPolicy := testutils.SelectPolicyByProvider(scenario, apiServerPolicy, apiServerPolicyForOCP) @@ -1325,7 +1343,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -1438,7 +1456,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) // nodeSelectors are merged @@ -1469,7 +1487,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) @@ -1483,7 +1501,7 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { } component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Containers[0].Args).To(ConsistOf([]string{ "--secure-port=5443", @@ -1955,7 +1973,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) rtest.ExpectResourceInList(deleteResources, "allow-apiserver", "calico-system", "networking.k8s.io", "v1", "NetworkPolicy") @@ -2051,7 +2069,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should not include deployment, service, SA, or PDB. Expect(rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment")).To(BeNil()) @@ -2094,7 +2112,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should render the correct resources. By("Checking each expected resource is actually rendered") @@ -2131,7 +2149,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) @@ -2147,7 +2165,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.Installation.ControlPlaneTolerations = []corev1.Toleration{tol} component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) }) @@ -2161,7 +2179,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2174,7 +2192,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.ForceHostNetwork = true component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) }) @@ -2187,7 +2205,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2204,7 +2222,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2219,7 +2237,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2232,7 +2250,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2246,7 +2264,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - _, _ = component.Objects() + _, _ = apiServerObjects(component) }) It("should render host networked with TKG provider", func() { @@ -2257,7 +2275,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2369,7 +2387,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2443,7 +2461,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) // nodeSelectors are merged Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(2)) @@ -2473,7 +2491,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(tol)) @@ -2485,7 +2503,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d).NotTo(BeNil()) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElement(corev1.Toleration{ @@ -2513,7 +2531,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(err).NotTo(HaveOccurred()) // Expect no UISettings / UISettingsGroups to be installed. - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) obj := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") Expect(obj).To(BeNil()) obj = rtest.GetResource(resources, "tigera-ui-user", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") @@ -2524,7 +2542,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) managedClusterAccessRole := rtest.GetResource(resources, render.MultiTenantManagedClustersAccessClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ @@ -2541,7 +2559,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) managedClusterAccessRole := rtest.GetResource(resources, render.ManagedClustersWatchClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ diff --git a/pkg/render/component.go b/pkg/render/component.go index eea911fbe3..93e4824f3a 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved. +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,3 +39,50 @@ type Component interface { // that create pods. Return OSTypeAny means that no node selector should be set for the "kubernetes.io/os" label. SupportedOSType() rmeta.OSType } + +// Extensible is implemented by components that expose extension points. The +// componentHandler uses ModifierKey() to look up registered modifiers. +// Components without extensions need not implement it. The method name is +// deliberately specific (not a generic Name()) so an unrelated method can't +// make a component modifier-eligible by accident. +type Extensible interface { + ModifierKey() string +} + +// ExtensionContextProvider is an optional companion to Extensible. A component +// implements it to hand its modifier component-specific context that can't be +// derived from the shared extensions.RenderContext - config only the component's +// controller has, such as a keypair the controller created. The componentHandler +// reads the returned value into RenderContext.Component before applying the +// modifier, and the modifier type-asserts it to the component's own context type. +type ExtensionContextProvider interface { + ExtensionContext() any +} + +// Component names used as keys into the extension modifier registry. Keep these +// in sync with the ModifierKey() methods that return them. +const ( + ComponentNameTypha = "typha" + ComponentNameNode = "node" + + // ComponentNameCNIPlugins keys the upstream CNI plugins image. The node + // component renders the cni-plugins init container, so the image resolves + // through its own override key. + ComponentNameCNIPlugins = "cni-plugins" + + // ComponentNameWindows keys the windows daemonset modifier. The two windows + // images resolve through their own override keys, since one component renders + // both. + ComponentNameWindows = "windows" + ComponentNameWindowsNodeImg = "windows-node-image" + ComponentNameWindowsCNIImg = "windows-cni-image" + + // ComponentNameKubeControllers keys the calico-kube-controllers modifier. The + // es-calico-kube-controllers deployment shares the component type but leaves + // its modifier key empty, so it is not decorated. + ComponentNameKubeControllers = "kube-controllers" + + // ComponentNameKubeControllersPolicy keys the calico-kube-controllers network + // policy modifier (the WAF admission webhook ingress rule). + ComponentNameKubeControllersPolicy = "kube-controllers-policy" +) diff --git a/pkg/render/decorate_helpers_test.go b/pkg/render/decorate_helpers_test.go new file mode 100644 index 0000000000..75f0587276 --- /dev/null +++ b/pkg/render/decorate_helpers_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +// stubExtComponent adapts raw object lists to a render.Component so a registered +// extension can be exercised through Set.Decorate, the same seam the component +// handler uses. key selects the extension; extCtx is delivered as the component's +// ExtensionContext (the typed config a RegisterModifier modifier reads). +type stubExtComponent struct { + key string + extCtx any + create, delete []client.Object +} + +func (s stubExtComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s stubExtComponent) Objects() ([]client.Object, []client.Object) { + return s.create, s.delete +} + +func (s stubExtComponent) Ready() bool { + return true +} + +func (s stubExtComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +func (s stubExtComponent) ModifierKey() string { + return s.key +} + +func (s stubExtComponent) ExtensionContext() any { + return s.extCtx +} + +// applyExtensions decorates a stub component holding the given objects with the +// extension registered under key, then renders it. For a modifier that needs the +// component's typed config, use applyExtensionsWithContext. +func applyExtensions(s *extensions.Set, key string, rc extensions.RenderContext, create, del []client.Object) ([]client.Object, []client.Object) { + return applyExtensionsWithContext(s, key, rc, nil, create, del) +} + +// applyExtensionsWithContext is applyExtensions for a modifier that reads the +// component's typed config: extCtx is delivered as the stub's ExtensionContext. +func applyExtensionsWithContext(s *extensions.Set, key string, rc extensions.RenderContext, extCtx any, create, del []client.Object) ([]client.Object, []client.Object) { + stub := stubExtComponent{key: key, extCtx: extCtx, create: create, delete: del} + return s.Decorate(stub, rc).Objects() +} diff --git a/pkg/render/enterprise_setup_test.go b/pkg/render/enterprise_setup_test.go new file mode 100644 index 0000000000..e896366124 --- /dev/null +++ b/pkg/render/enterprise_setup_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/extensions" +) + +// ext is the enterprise extension Set the render suite tests against. The +// Objects()-level image tests pass ext.Images() into the node/windows configs to +// pick up the enterprise images, and the enterprise modifier tests apply ext's +// modifiers explicitly to real render output to check they still match it. It is +// immutable once built and specs only read it, so a single instance is safe. +var ext *extensions.Set = enterprise.New() diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index 8fd24d35b6..f723469447 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -18,13 +18,9 @@ package render import ( "fmt" - "net" - "net/url" "golang.org/x/net/http/httpproxy" - operatorurl "github.com/tigera/operator/pkg/url" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" @@ -35,8 +31,6 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" @@ -45,7 +39,6 @@ import ( "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -85,27 +78,51 @@ func Guardian(cfg *GuardianConfiguration) Component { } } +// GuardianPolicy renders the guardian network policy. The core operator renders +// the OSS policy; the enterprise modifier (keyed ComponentNameGuardianPolicy) +// replaces it with the management-cluster policy. The error return is retained +// for callers but is always nil now that the fallible enterprise computation +// lives in the modifier. func GuardianPolicy(cfg *GuardianConfiguration) (Component, error) { - var policies []client.Object + return &guardianPolicyComponent{cfg: cfg}, nil +} - guardianAccessPolicy, err := guardianCalicoSystemPolicy(cfg) - if err != nil { - return nil, err - } - if guardianAccessPolicy != nil { - policies = []client.Object{ - guardianAccessPolicy, - } +const ComponentNameGuardianPolicy = "guardian-policy" + +type guardianPolicyComponent struct { + cfg *GuardianConfiguration +} + +func (c *guardianPolicyComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (c *guardianPolicyComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeAny } +func (c *guardianPolicyComponent) Ready() bool { return true } +func (c *guardianPolicyComponent) ModifierKey() string { return ComponentNameGuardianPolicy } + +// GuardianPolicyExtensionContext is the per-component context the guardian +// policy modifier reads (via RenderContext.Component). The enterprise guardian +// network policy is built entirely from these inputs. +type GuardianPolicyExtensionContext struct { + URL string + PodProxies []*httpproxy.Config + OpenShift bool + IncludeEgressNetworkPolicy bool +} + +func (c *guardianPolicyComponent) ExtensionContext() any { + return GuardianPolicyExtensionContext{ + URL: c.cfg.URL, + PodProxies: c.cfg.PodProxies, + OpenShift: c.cfg.OpenShift, + IncludeEgressNetworkPolicy: c.cfg.IncludeEgressNetworkPolicy, } +} - return NewPassthrough( - policies, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), - }, - ), nil +func (c *guardianPolicyComponent) Objects() ([]client.Object, []client.Object) { + return []client.Object{ossNetworkPolicy()}, []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), + } } // GuardianConfiguration contains all the config information needed to render the component. @@ -150,32 +167,40 @@ func (c *GuardianComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *GuardianComponent) ModifierKey() string { return GuardianName } + +// GuardianExtensionContext is the per-component context the guardian modifier +// reads (via RenderContext.Component). It carries the inputs the enterprise +// guardian behavior needs that a modifier can't derive from the installation: +// the management cluster's impersonation config, whether we're on OpenShift, +// and the trusted bundle mount path the CA env vars reference. +type GuardianExtensionContext struct { + OpenShift bool + Impersonation *operatorv1.Impersonation + TrustedBundleMountPath string +} + +func (c *GuardianComponent) ExtensionContext() any { + var impersonation *operatorv1.Impersonation + if c.cfg.ManagementClusterConnection != nil { + impersonation = c.cfg.ManagementClusterConnection.Spec.Impersonation + } + return GuardianExtensionContext{ + OpenShift: c.cfg.OpenShift, + Impersonation: impersonation, + TrustedBundleMountPath: c.cfg.TrustedCertBundle.MountPath(), + } +} + func (c *GuardianComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ - // common RBAC for EE and OSS c.serviceAccount(), c.clusterRole(), c.clusterRoleBinding(), - } - - if c.cfg.Installation.Variant.IsEnterprise() { - // Enterprise-specific RBAC and settings - objs = append(objs, - c.secretsRole(), - c.secretRoleBinding(), - // Install default UI settings for this managed cluster. - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), - ) - } - - objs = append(objs, c.deployment(), c.service(), secret.CopyToNamespace(GuardianNamespace, c.cfg.TunnelSecret)[0], - ) + } return objs, deprecatedObjects() } @@ -197,28 +222,6 @@ func (c *GuardianComponent) service() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - ports = append(ports, - corev1.ServicePort{ - Name: "elasticsearch", - Port: 9200, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - corev1.ServicePort{ - Name: "kibana", - Port: 5601, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - ) - } return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: GuardianServiceName, @@ -241,87 +244,42 @@ func (c *GuardianComponent) serviceAccount() *corev1.ServiceAccount { } func (c *GuardianComponent) clusterRole() *rbacv1.ClusterRole { - var policyRules []rbacv1.PolicyRule - if c.cfg.Installation.Variant.IsEnterprise() { - impersonation := c.cfg.ManagementClusterConnection.Spec.Impersonation - if impersonation != nil { - if impersonation.Users != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"users"}, - ResourceNames: impersonation.Users, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.Groups != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"groups"}, - ResourceNames: impersonation.Groups, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.ServiceAccounts != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - ResourceNames: impersonation.ServiceAccounts, - Verbs: []string{"impersonate"}, - }) - } - } - - policyRules = append(policyRules, rulesForManagementClusterRequests(c.cfg.OpenShift)...) - - if c.cfg.OpenShift { - policyRules = append(policyRules, rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.NonRootV2}, - }) - } - } else { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces", "services", "pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"apps"}, - Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "clusterinformations", - "tiers", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.stagedkubernetesnetworkpolicies", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "globalnetworksets", - "networksets", - }, - Verbs: []string{"get", "list", "watch"}, + policyRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"namespaces", "services", "pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "clusterinformations", + "tiers", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.stagedkubernetesnetworkpolicies", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "globalnetworksets", + "networksets", }, - ) + Verbs: []string{"get", "list", "watch"}, + }, } return &rbacv1.ClusterRole{ @@ -354,47 +312,6 @@ func (c *GuardianComponent) clusterRoleBinding() *rbacv1.ClusterRoleBinding { } } -// secretRole creates a Role that allows the management cluster to provision secrets to the tigera-operator Namespace. -// This is used to push secrets used by the managed cluster to access / authenticate with the management cluster. -func (c *GuardianComponent) secretsRole() *rbacv1.Role { - return &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRole, - Namespace: common.OperatorNamespace(), - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"create", "delete", "deletecollection", "update"}, - }, - }, - } -} - -func (c *GuardianComponent) secretRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRoleBindingName, - Namespace: common.OperatorNamespace(), - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: GuardianSecretsRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - func (c *GuardianComponent) deployment() *appsv1.Deployment { var replicas int32 = 1 @@ -468,14 +385,6 @@ func (c *GuardianComponent) container() []corev1.Container { } envVars = append(envVars, c.cfg.Installation.Proxy.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - envVars = append(envVars, - corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - ) - } - if c.cfg.GuardianClientKeyPair != nil { envVars = append(envVars, corev1.EnvVar{ @@ -576,198 +485,6 @@ func ossNetworkPolicy() *v3.NetworkPolicy { } } -func guardianCalicoSystemPolicy(cfg *GuardianConfiguration) (*v3.NetworkPolicy, error) { - if !cfg.Installation.Variant.IsEnterprise() { - return ossNetworkPolicy(), nil - } - - egressRules := []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: PacketCaptureEntityRule, - }, - } - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.OpenShift) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.KubeAPIServerEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.PrometheusEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: TigeraAPIServerEntityRule, - }, - }...) - - // The loop below creates an egress rule for each unique destination that the Guardian pods connect to. If there are - // multiple guardian pods and their proxy settings differ, then there are multiple destinations that must have egress allowed. - allowedDestinations := map[string]bool{} - processedPodProxies := ProcessPodProxies(cfg.PodProxies) - for _, podProxyConfig := range processedPodProxies { - var proxyURL *url.URL - var err error - if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { - targetURL := &url.URL{ - // The scheme should be HTTPS, as we are establishing an mTLS session with the target. - Scheme: "https", - - // We expect `target` to be of the form host:port. - Host: cfg.URL, - } - - proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) - if err != nil { - return nil, err - } - } - - var tunnelDestinationHostPort string - if proxyURL != nil { - proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) - if err != nil { - return nil, err - } - - tunnelDestinationHostPort = proxyHostPort - } else { - // cfg.URL has host:port form - tunnelDestinationHostPort = cfg.URL - } - - // Check if we've already created an egress rule for this destination. - if allowedDestinations[tunnelDestinationHostPort] { - continue - } - - host, port, err := net.SplitHostPort(tunnelDestinationHostPort) - if err != nil { - return nil, err - } - parsedPort, err := numorstring.PortFromString(port) - if err != nil { - return nil, err - } - parsedIp := net.ParseIP(host) - if parsedIp == nil { - // Domain-based egress rules require the EgressAccessControl license feature. - if !cfg.IncludeEgressNetworkPolicy { - continue - } - // Assume host is a valid hostname. - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Domains: []string{host}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - - } else { - var netSuffix string - if parsedIp.To4() != nil { - netSuffix = "/32" - } else { - netSuffix = "/128" - } - - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Nets: []string{parsedIp.String() + netSuffix}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - } - } - - egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) - - guardianIngressDestinationEntityRule := v3.EntityRule{Ports: networkpolicy.Ports(GuardianTargetPort)} - networkpolicyHelper := networkpolicy.DefaultHelper() - var ingressRules []v3.Rule - if cfg.Installation.Variant.IsEnterprise() { - ingressRules = append(ingressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: FluentdSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceBenchmarkerSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceReporterSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceSnapshotterSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: networkpolicyHelper.ComplianceControllerSourceEntityRule(), - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionInstallerSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: guardianIngressDestinationEntityRule, - }, - }...) - } - - policy := &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianPolicyName, - Namespace: GuardianNamespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(GuardianName), - Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - Ingress: ingressRules, - Egress: egressRules, - }, - } - - return policy, nil -} - func ProcessPodProxies(podProxies []*httpproxy.Config) []*httpproxy.Config { // If pod proxies are empty, then pod proxy resolution has not yet occurred. // Assume that a single Guardian pod is running without a proxy. @@ -782,304 +499,6 @@ func GuardianService(clusterDomain string) string { return fmt.Sprintf("https://%s.%s.svc.%s:%d", GuardianServiceName, GuardianNamespace, clusterDomain, 443) } -// rulesForManagementClusterRequests returns the set of RBAC rules needed by Guardian in order to -// satisfy requests from the management cluster over the tunnel. -func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - // Common rules required to handle requests from multiple components in the management cluster. - { - // ID uses read-only permissions and kube-controllers uses both read and write verbs. - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Allows Linseed to watch namespaces before copying its token. - // Also enables PolicyRecommendation to watch namespaces, - // and Manager/kube-controllers to list them. - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Nodes to monitor for deletions. - // Manager performs a list operation on Nodes. - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Pods to verify existence for IPAM garbage collection. - // Manager performs get operations on Pods. - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needed by kube-controllers to validate licenses; also used by ID. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation & ID uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworksets", - "networkpolicies", - "tier.networkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed by guardian to handle manager authorization reviews. - { - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, - Verbs: []string{"list", "get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings", "uisettingsgroups"}, - Verbs: []string{"list", "get"}, - }, - - // Rules needed by guardian to handle other manager requests. - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - // Allow query server talk to Prometheus via the manager user. - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "calico-node-prometheus:9090", - "https:calico-api:8080", - }, - Verbs: []string{"create", "get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"daemonsets", "replicasets", "statefulsets"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"authentication.k8s.io"}, - Resources: []string{"tokenreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"authorization.k8s.io"}, - Resources: []string{"subjectaccessreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - ResourceNames: []string{"default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworkpolicies", - "networksets", - "stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.globalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"list"}, - }, - - // Rules needed by guardian to handle policy recommendation requests. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "policyrecommendationscopes", - "policyrecommendationscopes/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - - // Rules needed by guardian to handle calico-kube-controller requests. - { - // Nodes are watched to monitor for deletions. - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/status"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needs to manage hostendpoints. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Needs access to update clusterinformations. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Needs to manipulate kubecontrollersconfiguration, which contains its config. - // It creates a default if none exists, and updates status as well. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"kubecontrollersconfigurations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Grant permissions to access ClusterInformation resources in managed clusters. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - - // Rules needed by guardian to handle Intrusion detection requests. - { - APIGroups: []string{""}, - Resources: []string{"podtemplates"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalalerts", - "globalalerts/status", - "globalthreatfeeds", - "globalthreatfeeds/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed to fetch the compliance reports - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes", "globalreports"}, - Verbs: []string{"get", "list", "watch"}, - }, - } - - // Rules needed by policy recommendation in openshift. - if isOpenShift { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, - }, - ) - } - - return rules -} - func deprecatedObjects() []client.Object { return []client.Object{ // All the Guardian objects were moved to "calico-system" circa Calico v3.30, and so the legacy tigera-guardian diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 7e82472a41..16f7653031 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -36,6 +36,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -43,6 +44,21 @@ import ( "github.com/tigera/operator/pkg/render/testutils" ) +// guardianObjects renders the guardian component and applies the registered +// enterprise modifier the way the componentHandler does. +func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { + g := render.Guardian(cfg) + ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) + objs, _ := g.Objects() + rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any + if p, ok := g.(render.ExtensionContextProvider); ok { + extCtx = p.ExtensionContext() + } + out, _ := applyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, objs, nil) + return out +} + var _ = Describe("Rendering tests", func() { var cfg *render.GuardianConfiguration var g render.Component @@ -95,6 +111,14 @@ var _ = Describe("Rendering tests", func() { g = render.Guardian(cfg) Expect(g.ResolveImages(nil)).To(BeNil()) resources, deleteResources = g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so these enterprise tests exercise the integrated output. + rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any + if p, ok := g.(render.ExtensionContextProvider); ok { + extCtx = p.ExtensionContext() + } + resources, _ = applyExtensionsWithContext(ext, render.GuardianName, rc, extCtx, resources, nil) } BeforeEach(func() { @@ -189,8 +213,7 @@ var _ = Describe("Rendering tests", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -230,8 +253,7 @@ var _ = Describe("Rendering tests", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -270,8 +292,7 @@ var _ = Describe("Rendering tests", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) @@ -301,9 +322,7 @@ var _ = Describe("Rendering tests", func() { It("should render SecurityContextConstrains properly when provider is OpenShift", func() { cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift cfg.OpenShift = true - component := render.Guardian(cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := guardianObjects(cfg) role := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ @@ -327,7 +346,16 @@ var _ = Describe("Rendering tests", func() { cfg.IncludeEgressNetworkPolicy = includeEgressNetworkPolicy g, err := render.GuardianPolicy(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ = g.Objects() + objs, _ := g.Objects() + // Apply the registered enterprise modifier the way the componentHandler + // does, so the enterprise policy is exercised. For the Calico variant the + // modifier is a no-op and the OSS policy is returned. + rc := extensions.RenderContext{Installation: cfg.Installation} + var extCtx any + if p, ok := g.(render.ExtensionContextProvider); ok { + extCtx = p.ExtensionContext() + } + resources, _ = applyExtensionsWithContext(ext, render.ComponentNameGuardianPolicy, rc, extCtx, objs, nil) } Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { @@ -441,8 +469,7 @@ var _ = Describe("guardian", func() { } }) It("should render when disabled", func() { - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -452,8 +479,7 @@ var _ = Describe("guardian", func() { It("should render when set to disabled", func() { cfg.TunnelCAType = operatorv1.CATypeTigera - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -464,8 +490,7 @@ var _ = Describe("guardian", func() { It("should render when enabled", func() { cfg.TunnelCAType = operatorv1.CATypePublic - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -503,8 +528,7 @@ var _ = Describe("guardian", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment, ok := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index 1f592f8358..36bc0068da 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -16,7 +16,6 @@ package kubecontrollers import ( "fmt" - "path/filepath" "slices" "strconv" "strings" @@ -36,17 +35,13 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/pkg/url" ) const ( @@ -57,33 +52,12 @@ const ( KubeControllerMetrics = "calico-kube-controllers-metrics" KubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "kube-controller-access" - // WASMPullSecretName is the dedicated image-pull Secret (a renamed copy of - // the install pull secret) that the WAF reconciler replicates into tenant - // namespaces for the Coraza wasm OCI pull. A dedicated name avoids clashing - // with the operator-managed tigera-pull-secret the GatewayAPI render also - // copies into those namespaces (EV-6386). - WASMPullSecretName = "tigera-waf-pull-secret" - - // WASMCACertName is the dedicated CA-bundle ConfigMap (in the controller - // namespace) the WAF reconciler replicates into tenant namespaces for the - // Coraza wasm OCI registry TLS check — a dedicated name avoids clashing with - // the operator-managed tigera-ca-bundle ConfigMap the GatewayAPI render also - // copies there (EV-6386). The source copy is a renamed copy of the trusted - // bundle, provisioned by the core controller and passed in as WASMCACert. - WASMCACertName = "tigera-waf-ca-bundle" - - EsKubeController = "es-calico-kube-controllers" - EsKubeControllerRole = "es-calico-kube-controllers" - EsKubeControllerRoleBinding = "es-calico-kube-controllers" - EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" - EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + // ManagedClustersWatchRoleBindingName binds kube-controllers to the managed-cluster + // watch ClusterRole. Used by both calico-kube-controllers (in a management cluster) + // and the enterprise es-calico-kube-controllers, so the binding stays generic here. ManagedClustersWatchRoleBindingName = "es-calico-kube-controllers-managed-cluster-watch" - ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" - ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" - ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" - ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" - KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" + KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" // KubeControllersHealthPort is the port the kube-controllers HealthAggregator listens on when run from the // combined calico binary. The legacy per-component image uses file-based health checks instead. @@ -99,9 +73,6 @@ type KubeControllersConfiguration struct { ManagementClusterConnection *operatorv1.ManagementClusterConnection Authentication *operatorv1.Authentication - // Whether or not the LogStorage CRD is present in the cluster. - LogStorageExists bool - ClusterDomain string MetricsPort int @@ -112,12 +83,8 @@ type KubeControllersConfiguration struct { // namespace to be returned by the rendered. Expected that the calling code // take care to pass the same secret on each reconcile where possible. KubeControllersGatewaySecret *corev1.Secret - WASMPullSecret *corev1.Secret - WASMCACert *corev1.ConfigMap TrustedBundle certificatemanagement.TrustedBundleRO - MetricsServerTLS certificatemanagement.KeyPairInterface - // Namespace to be installed into. Namespace string @@ -128,28 +95,51 @@ type KubeControllersConfiguration struct { // If this is nil, then we should run in zero-tenant mode. Tenant *operatorv1.Tenant - // WAFGatewayExtensionEnabled gates the WAF v3 (Gateway API add-on) surface - // on calico-kube-controllers: the applicationlayer controller enablement, - // the WAF / Gateway-API / EnvoyExtensionPolicy / event / secret-replication - // RBAC, the WASM_IMAGE / WASM_PULL_SECRET / WASM_CA_CERT env vars, and the - // gateway envoy-proxy wasm image resolution. Sourced from - // `GatewayAPI.spec.extensions.waf.state == Enabled` (default off). - // See design `tigera/designs#25` (PMREQ-384). - WAFGatewayExtensionEnabled bool - - // WAFWebhookServerTLS is the serving certificate for the in-process WAF - // SecLang validating admission webhook hosted by calico-kube-controllers. - // When set (WAF enabled), it is mounted into the Pod and the webhook server - // reads it from WAF_WEBHOOK_CERT_DIR. Issued for the tigera-waf-webhook - // Service DNS name. Nil leaves the Deployment untouched (and the in-process - // server self-disables when the cert is absent). - WAFWebhookServerTLS certificatemanagement.KeyPairInterface - - // WAFWebhookCABundle is the PEM of the CA that issued WAFWebhookServerTLS - // (the operator CA), stamped into the ValidatingWebhookConfiguration's - // caBundle so the apiserver can verify the in-process webhook endpoint. - // Only consulted when WAFGatewayExtensionEnabled is true. - WAFWebhookCABundle []byte + // The fields below parameterize the generic kube-controllers component. The + // variant assemblers (NewCalicoKubeControllers, the enterprise es builder) + // fill them; the component renders them without any variant or component-name + // branching. + + // Name is the deployment / pod / container name (and the value the metrics + // Service selects on). + Name string + // ConfigName is the KUBE_CONTROLLERS_CONFIG_NAME the binary reconciles. + ConfigName string + // RoleName / RoleBindingName / MetricsName name the ClusterRole, its binding, + // and the Prometheus metrics Service. + RoleName string + RoleBindingName string + MetricsName string + // EnabledControllers is the ENABLED_CONTROLLERS list. The deployment is only + // rendered when it is non-empty. + EnabledControllers []string + // Rules are the ClusterRole policy rules. + Rules []rbacv1.PolicyRule + // NetworkPolicy, when set, is rendered into the install namespace (and the + // deprecated allow-tigera policy named DeprecatedNetworkPolicyName is deleted). + NetworkPolicy *v3.NetworkPolicy + DeprecatedNetworkPolicyName string + // ExtraEnv is appended to the deployment's container env. + ExtraEnv []corev1.EnvVar + // DisableConfigAPI sets DISABLE_KUBE_CONTROLLERS_CONFIG_API. + DisableConfigAPI bool + + // ModifierKey is the extension modifier key the component reports through + // render.Extensible. calico-kube-controllers sets it so the enterprise modifier + // can layer on the enterprise surface; es-calico-kube-controllers leaves it empty + // so it is never decorated. + ModifierKey string +} + +// calicoKubeControllersPolicyComponent wraps the calico-kube-controllers network +// policy passthrough so it is render.Extensible: the enterprise modifier adds the WAF +// admission webhook ingress rule. The base policy carries no WAF. +type calicoKubeControllersPolicyComponent struct { + render.Component +} + +func (calicoKubeControllersPolicyComponent) ModifierKey() string { + return render.ComponentNameKubeControllersPolicy } func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { @@ -159,106 +149,40 @@ func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDe toCreate = append(toCreate, defaultDeny) } - return render.NewPassthrough( + return calicoKubeControllersPolicyComponent{render.NewPassthrough( toCreate, []client.Object{ // allow-tigera Tier was renamed to calico-system networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("kube-controller-access", cfg.Namespace), networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", common.CalicoNamespace), }, - ) + )} } -func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) - enabledControllers := []string{"node", "loadbalancer"} - if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"watch", "list", "get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "update", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, - }, - ) - enabledControllers = append(enabledControllers, "service", "federatedservices", "usage") - if cfg.WAFGatewayExtensionEnabled { - enabledControllers = append(enabledControllers, "applicationlayer") - } - } - - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: KubeControllerRole, - kubeControllerRoleBindingName: KubeControllerRoleBinding, - kubeControllerName: KubeController, - kubeControllerConfigName: "default", - kubeControllerMetricsName: KubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - enabledControllers: enabledControllers, - } +// NewKubeControllers builds a kube-controllers component from a fully-populated +// configuration. Callers (NewCalicoKubeControllers, the enterprise es-kube-controllers +// builder) fill the generic Name/Rules/EnabledControllers/ExtraEnv/NetworkPolicy fields; +// the component renders them with no variant branching. +func NewKubeControllers(cfg *KubeControllersConfiguration) render.Component { + return &kubeControllersComponent{cfg: cfg} } -func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - var kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) - - if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, - rbacv1.PolicyRule{ - APIGroups: []string{"elasticsearch.k8s.elastic.co"}, - Resources: []string{"elasticsearches"}, - Verbs: []string{"watch", "get", "list"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings"}, - Verbs: []string{"watch", "list", "get"}, - }, - ) - - kubeControllerCalicoSystemPolicy = esKubeControllersCalicoSystemPolicy(cfg) - } - - var enabledControllers []string - if !cfg.Tenant.MultiTenant() { - // Zero and single tenant cluster needs elasticsearch configuration - enabledControllers = append(enabledControllers, "authorization", "elasticsearchconfiguration") - if cfg.ManagementCluster != nil && cfg.Tenant == nil { - // Enterprise will require the managedcluster controller to push licenses - enabledControllers = append(enabledControllers, "managedcluster") - } - } - - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: EsKubeControllerRole, - kubeControllerRoleBindingName: EsKubeControllerRoleBinding, - kubeControllerName: EsKubeController, - kubeControllerConfigName: "elasticsearch", - kubeControllerMetricsName: EsKubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - kubeControllerCalicoSystemPolicy: kubeControllerCalicoSystemPolicy, - enabledControllers: enabledControllers, - } +// NewCalicoKubeControllers builds the calico-kube-controllers component. The base is +// pure OSS (the common rules plus the node and loadbalancer controllers); the Calico +// Enterprise additions (extra RBAC, enterprise controllers, metrics TLS, the WAF v3 +// surface) are layered on by the enterprise modifier keyed by ModifierKey. +func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) render.Component { + cfg.Name = KubeController + cfg.ConfigName = "default" + cfg.RoleName = KubeControllerRole + cfg.RoleBindingName = KubeControllerRoleBinding + cfg.MetricsName = KubeControllerMetrics + cfg.ModifierKey = render.ComponentNameKubeControllers + + cfg.Rules = KubeControllersRoleCommonRules(cfg) + cfg.EnabledControllers = []string{"node", "loadbalancer"} + + return NewKubeControllers(cfg) } type kubeControllersComponent struct { @@ -267,24 +191,6 @@ type kubeControllersComponent struct { // Internal state generated by the given configuration. calicoImage string - - kubeControllerServiceAccountName string - kubeControllerRoleName string - kubeControllerRoleBindingName string - kubeControllerName string - kubeControllerConfigName string - kubeControllerMetricsName string - - kubeControllersRules []rbacv1.PolicyRule - kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - - enabledControllers []string - - // wasmImage is the fully-resolved OCI reference for the Coraza WAF wasm - // binary (Enterprise only). Surfaced to the kube-controllers binary via - // the WASM_IMAGE env var; consumed by the applicationlayer reconcilers - // in tigera/calico-private to program WAF policy attachments. - wasmImage string } func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error { @@ -296,15 +202,6 @@ func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error if err != nil { return err } - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.WAFGatewayExtensionEnabled { - // The Coraza WAF wasm is baked into the gateway envoy-proxy image as its - // final layer; Envoy Gateway extracts it from there. Point WASM_IMAGE at - // that same image (no standalone coraza-wasm image needed). - c.wasmImage, err = components.GetReference(components.ComponentGatewayAPIEnvoyProxy, reg, path, prefix, is) - if err != nil { - return err - } - } return nil } @@ -316,12 +213,14 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate := []client.Object{} objectsToDelete := []client.Object{} - if c.kubeControllerCalicoSystemPolicy != nil { - objectsToCreate = append(objectsToCreate, c.kubeControllerCalicoSystemPolicy) - // allow-tigera Tier was renamed to calico-system - objectsToDelete = append(objectsToDelete, - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("es-kube-controller-access", c.cfg.Namespace), - ) + if c.cfg.NetworkPolicy != nil { + objectsToCreate = append(objectsToCreate, c.cfg.NetworkPolicy) + if c.cfg.DeprecatedNetworkPolicyName != "" { + // allow-tigera Tier was renamed to calico-system + objectsToDelete = append(objectsToDelete, + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject(c.cfg.DeprecatedNetworkPolicyName, c.cfg.Namespace), + ) + } } objectsToCreate = append(objectsToCreate, @@ -331,7 +230,7 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) ) objectsToCreate = append(objectsToCreate, c.managedClusterRoleBindings()...) - if len(c.enabledControllers) > 0 { + if len(c.cfg.EnabledControllers) > 0 { // There's something to run, so create the deployment. objectsToCreate = append(objectsToCreate, c.controllersDeployment()) } else { @@ -346,26 +245,6 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( secret.CopyToNamespace(c.cfg.Namespace, c.cfg.KubeControllersGatewaySecret)...)...) } - if c.cfg.WASMPullSecret != nil { - objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( - secret.CopyToNamespace(c.cfg.Namespace, c.cfg.WASMPullSecret)...)...) - } - if c.cfg.WASMCACert != nil { - objectsToCreate = append(objectsToCreate, c.cfg.WASMCACert) - } - - // The in-process WAF admission webhook surface (Service fronting this Pod + - // ValidatingWebhookConfiguration). Rendered here, rather than as a - // passthrough in the core controller, so the objects are cleaned up when the - // WAF extension is disabled or the GatewayAPI CR is removed. - if c.kubeControllerName == KubeController { - webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(c.cfg.WAFWebhookCABundle) - if c.cfg.WAFGatewayExtensionEnabled { - objectsToCreate = append(objectsToCreate, webhookObjs...) - } else { - objectsToDelete = append(objectsToDelete, webhookObjs...) - } - } if c.cfg.MetricsPort != 0 { objectsToCreate = append(objectsToCreate, c.prometheusService()) @@ -385,7 +264,13 @@ func (c *kubeControllersComponent) Ready() bool { return true } -func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { +// ModifierKey implements render.Extensible. It is empty for es-calico-kube-controllers +// (never decorated) and set for calico-kube-controllers. +func (c *kubeControllersComponent) ModifierKey() string { + return c.cfg.ModifierKey +} + +func KubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { // Nodes are watched to monitor for deletions. @@ -511,158 +396,11 @@ func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1. return rules } -func kubeControllersRoleEnterpriseCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - }, - { - // Needed to validate the license - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - // Needed to update the status of the LicenseKey with the result of license validation. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures/status"}, - Verbs: []string{"update"}, - }, - } - - if cfg.WAFGatewayExtensionEnabled { - // WAF v3 (Gateway API add-on) RBAC. Gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. - rules = append(rules, - // Application-layer (gateway-addons) reconcilers reconcile WAF resources - // against Gateway API targetRefs and emit events on the policy objects. - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - }, - rbacv1.PolicyRule{ - // Validate Gateway API targetRefs and surface attachment status. - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - }, - // controller-runtime Reconcilers (e.g. the applicationlayer manager) record - // events on watched objects via Recorder.Eventf; both core and events.k8s.io - // API groups are emitted depending on the kubernetes version. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - // Application-layer reconciler replicates the WAF wasm pull Secret from - // the controller namespace (calico-system) into each WAFPolicy's - // namespace so the rendered EnvoyExtensionPolicy can reference it. Also - // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF - // targetRef to bind the Coraza wasm filter at the gateway / route. - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler stamps each namespace with its - // allocated WAF rule-id range (applicationlayer.projectcalico.org/waf-id-range - // annotation) so application operators can author in-range rules. The - // base role already grants namespaces get/list/watch; the annotation - // write needs patch/update, gated to the WAF path. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - }, - ) - } - - if cfg.ManagementClusterConnection != nil { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "create", "update", "list", "watch"}, - }, - ) - } - - return rules -} - func (c *kubeControllersComponent) controllersServiceAccount() *corev1.ServiceAccount { return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, Labels: map[string]string{}, }, @@ -673,9 +411,9 @@ func (c *kubeControllersComponent) controllersClusterRole() *rbacv1.ClusterRole role := &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, - Rules: c.kubeControllersRules, + Rules: c.cfg.Rules, } return role @@ -698,7 +436,7 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: KubeController, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, }, }, @@ -707,95 +445,20 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { env := []corev1.EnvVar{ - {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.kubeControllerConfigName}, + {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.cfg.ConfigName}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.enabledControllers, ",")}, - {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.Tenant.MultiTenant() && c.kubeControllerConfigName == "elasticsearch")}, + {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.cfg.EnabledControllers, ",")}, + {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.DisableConfigAPI)}, } env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) + env = append(env, c.cfg.ExtraEnv...) - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Tenant != nil { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) - } - - if c.kubeControllerName == EsKubeController { - // What started as a workaround is now the default behaviour. This feature uses our backend in order to - // log into Kibana for users from external identity providers, rather than configuring an authn realm - // in the Elastic stack. - env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) - - if c.cfg.Authentication != nil { - env = append(env, - corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: c.cfg.Authentication.Spec.UsernamePrefix}, - corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: c.cfg.Authentication.Spec.GroupsPrefix}, - ) - } - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: c.cfg.TrustedBundle.MountPath()}) - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - // Application-layer (gateway-addons / WAF v3) env vars, gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. When the gate is - // off (default), none of the WASM_* env vars are rendered and the - // kube-controllers binary skips the WAF reconcilers entirely (see the - // applicationlayer entry in enabledControllers). - if c.cfg.WAFGatewayExtensionEnabled { - // Application-layer (gateway-addons) reconcilers consume the Coraza WAF - // wasm OCI reference from this env var to program WAF policy attachments. - // Empty when ResolveImages was not called for the Calico variant; the - // reconciler stamps Programmed=False/WASMUnavailable in that case. - if c.wasmImage != "" { - env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: c.wasmImage}) - } - - // WASM_PULL_SECRET names the imagePullSecret the reconciler replicates - // from the kube-controllers namespace into a WAFPolicy's namespace so - // the rendered EnvoyExtensionPolicy can pull the wasm OCI artifact from - // a private Tigera registry. Source the name from the first - // Installation.ImagePullSecrets entry so multi-tenant / BYO-registry - // installs reuse whatever pull secret operator already attaches here. - if c.cfg.WASMPullSecret != nil { - env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: c.cfg.WASMPullSecret.Name}) - } - - // WASM_CA_CERT names the dedicated CA bundle ConfigMap (provisioned as - // WASMCACert) that the reconciler replicates alongside WASM_PULL_SECRET - // so the EnvoyExtensionPolicy wasm fetcher trusts the registry's TLS - // chain. Only set when the source ConfigMap is actually rendered. - if c.cfg.WASMCACert != nil { - env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: c.cfg.WASMCACert.Name}) - } - } - } - - if c.cfg.MetricsServerTLS != nil { - env = append(env, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, - ) - } if c.cfg.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "CA_CRT_PATH", Value: c.cfg.TrustedBundle.MountPath()}, ) } - if c.cfg.WAFWebhookServerTLS != nil { - // The in-process WAF admission webhook server (calico-private - // applicationlayer manager) reads its serving cert (tls.crt/tls.key) - // from this directory; the controller-runtime webhook server only - // registers when the cert is present. - env = append(env, - corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(c.cfg.WAFWebhookServerTLS.VolumeMountCertificateFilePath())}, - ) - } // UID 999 is used in kube-controller Dockerfile. sc := securitycontext.NewNonRootContext() @@ -828,7 +491,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { } container := corev1.Container{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Image: c.calicoImage, Command: containerCommand, Env: env, @@ -839,34 +502,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { VolumeMounts: c.kubeControllersVolumeMounts(), } - if c.cfg.WAFWebhookServerTLS != nil { - // Expose the in-process WAF admission-webhook port that the - // tigera-waf-webhook Service forwards to. - container.Ports = append(container.Ports, corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: applicationlayer.WAFWebhookContainerPort, - Protocol: corev1.ProtocolTCP, - }) - } - - if c.kubeControllerName == EsKubeController && !c.cfg.Tenant.MultiTenant() { - _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, render.ElasticsearchNamespace)) - container.Env = append(container.Env, []corev1.EnvVar{ - relasticsearch.ElasticHostEnvVar(esHost), - relasticsearch.ElasticPortEnvVar(esPort), - relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticCAEnvVar(c.SupportedOSType()), - }...) - } - var initContainers []corev1.Container - if c.cfg.MetricsServerTLS != nil && c.cfg.MetricsServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.MetricsServerTLS.InitContainer(c.cfg.Namespace, sc)) - } - if c.cfg.WAFWebhookServerTLS != nil && c.cfg.WAFWebhookServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.WAFWebhookServerTLS.InitContainer(c.cfg.Namespace, sc)) - } tolerations := appendUniqueTolerations(c.cfg.Installation.ControlPlaneTolerations, rmeta.TolerateCriticalAddonsAndControlPlane...) if c.cfg.Installation.KubernetesProvider.IsGKE() { tolerations = appendUniqueTolerations(tolerations, rmeta.TolerateGKEARM64NoSchedule) @@ -875,7 +511,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { NodeSelector: c.cfg.Installation.ControlPlaneNodeSelector, Tolerations: tolerations, ImagePullSecrets: c.cfg.Installation.ImagePullSecrets, - ServiceAccountName: c.kubeControllerServiceAccountName, + ServiceAccountName: KubeControllerServiceAccount, InitContainers: initContainers, Containers: []corev1.Container{container}, Volumes: c.kubeControllersVolumes(), @@ -886,7 +522,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { d := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, }, Spec: appsv1.DeploymentSpec{ @@ -896,7 +532,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, Annotations: c.annotations(), }, @@ -928,20 +564,20 @@ func (c *kubeControllersComponent) controllersClusterRoleBinding() *rbacv1.Clust for _, ns := range c.cfg.BindingNamespaces { subjects = append(subjects, rbacv1.Subject{ Kind: "ServiceAccount", - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: ns, }) } return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleBindingName, + Name: c.cfg.RoleBindingName, Labels: map[string]string{}, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, Subjects: subjects, } @@ -950,7 +586,7 @@ func (c *kubeControllersComponent) controllersClusterRoleBinding() *rbacv1.Clust func (c *kubeControllersComponent) managedClusterRoleBindings() []client.Object { if c.cfg.ManagementCluster != nil { return []client.Object{ - rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, c.kubeControllerServiceAccountName, []string{c.cfg.Namespace}), + rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, KubeControllerServiceAccount, []string{c.cfg.Namespace}), } } return []client.Object{} @@ -962,16 +598,16 @@ func (c *kubeControllersComponent) prometheusService() *corev1.Service { return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerMetricsName, + Name: c.cfg.MetricsName, Namespace: c.cfg.Namespace, Annotations: map[string]string{ "prometheus.io/scrape": "true", "prometheus.io/port": fmt.Sprintf("%d", c.cfg.MetricsPort), }, - Labels: map[string]string{"k8s-app": c.kubeControllerName}, + Labels: map[string]string{"k8s-app": c.cfg.Name}, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": c.kubeControllerName}, + Selector: map[string]string{"k8s-app": c.cfg.Name}, // "Headless" service; prevent kube-proxy from rendering any rules for this service // (which is only intended for Prometheus to scrape). ClusterIP: "None", @@ -1000,9 +636,6 @@ func (c *kubeControllersComponent) annotations() map[string]string { am = make(map[string]string) } - if c.cfg.MetricsServerTLS != nil { - am[c.cfg.MetricsServerTLS.HashAnnotationKey()] = c.cfg.MetricsServerTLS.HashAnnotationValue() - } if c.cfg.KubeControllersGatewaySecret != nil { am[render.ElasticsearchUserHashAnnotation] = rmeta.AnnotationHash(c.cfg.KubeControllersGatewaySecret.Data) } @@ -1014,12 +647,6 @@ func (c *kubeControllersComponent) kubeControllersVolumeMounts() []corev1.Volume if c.cfg.TrustedBundle != nil { mounts = append(mounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) } - if c.cfg.MetricsServerTLS != nil { - mounts = append(mounts, c.cfg.MetricsServerTLS.VolumeMount(c.SupportedOSType())) - } - if c.cfg.WAFWebhookServerTLS != nil { - mounts = append(mounts, c.cfg.WAFWebhookServerTLS.VolumeMount(c.SupportedOSType())) - } return mounts } @@ -1028,12 +655,6 @@ func (c *kubeControllersComponent) kubeControllersVolumes() []corev1.Volume { if c.cfg.TrustedBundle != nil { volumes = append(volumes, c.cfg.TrustedBundle.Volume()) } - if c.cfg.MetricsServerTLS != nil { - volumes = append(volumes, c.cfg.MetricsServerTLS.Volume()) - } - if c.cfg.WAFWebhookServerTLS != nil { - volumes = append(volumes, c.cfg.WAFWebhookServerTLS.Volume()) - } return volumes } @@ -1076,20 +697,6 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }) } - // Allow the kube-apiserver to reach the in-process WAF admission webhook on - // :9443 (EV-6386). render-v3 wires the webhook Service/config/cert + the - // server, but without this ingress rule the calico-system default-deny drops - // the apiserver→:9443 call and every WAFPolicy/WAFPlugin admission times out. - if cfg.WAFGatewayExtensionEnabled { - ingressRules = append(ingressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), - }, - }) - } - if r, err := cfg.K8sServiceEp.DestinationEntityRule(); r != nil && err == nil { egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, @@ -1114,53 +721,3 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }, } } - -func esKubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.NetworkPolicy { - if cfg.ManagementClusterConnection != nil { - return nil - } - - egressRules := []v3.Rule{} - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(443, 6443, 12388), - }, - }, - }...) - - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), - }, - }...) - - networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicyHelper.ManagerEntityRule(), - }, - }...) - - return &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EsKubeControllerNetworkPolicyName, - Namespace: cfg.Namespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(EsKubeController), - Types: []v3.PolicyType{v3.PolicyTypeEgress}, - Egress: egressRules, - }, - } -} diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 9f30802d99..89cfad0940 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -16,12 +16,10 @@ package kubecontrollers_test import ( "fmt" - "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -41,14 +39,13 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" - "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -231,175 +228,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateCriticalAddonsAndControlPlane)) }) - It("should render all calico kube-controllers resources for a default configuration (standalone) using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.WASMPullSecretName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.WASMCACertName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - instance.Variant = operatorv1.CalicoEnterprise - instance.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "tigera-pull-secret"}} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - cfg.WAFWebhookCABundle = []byte("fake-ca-bundle") - // core_controller provisions a dedicated WAF wasm pull secret (a renamed - // copy of the install pull secret) so the reconciler can replicate it into - // WAFPolicy namespaces without clashing with the operator-managed - // tigera-pull-secret; surface it here so it renders and WASM_PULL_SECRET is set. - cfg.WASMPullSecret = &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMPullSecretName, Namespace: common.CalicoNamespace}} - // Likewise core_controller provisions the dedicated WAF wasm CA-bundle - // ConfigMap (a renamed copy of the trusted bundle); surface it here so it - // renders and WASM_CA_CERT is set. - cfg.WASMCACert = &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMCACertName, Namespace: common.CalicoNamespace}} - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - Expect(dp.Spec.Template.Spec.ImagePullSecrets).To(ContainElement(corev1.LocalObjectReference{Name: "tigera-pull-secret"})) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - // Application-layer reconcilers consume these env vars to program WAF - // EnvoyExtensionPolicy attachments. - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, - })) - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_PULL_SECRET", Value: kubecontrollers.WASMPullSecretName, - })) - // WASM_CA_CERT names the dedicated WAF trusted-bundle ConfigMap that the - // reconciler replicates into WAFPolicy namespaces (kept separate from the - // operator-managed tigera-ca-bundle the GatewayAPI render also copies there). - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_CA_CERT", Value: kubecontrollers.WASMCACertName, - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - - clusterRole := rtest.GetResource(resources, kubecontrollers.KubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(38), "cluster role should have 38 rules") - - // Application-layer reconciler RBAC: WAF CRDs (resources, /status, /finalizers). - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - })) - // Gateway API targetRef validation + status patching. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - })) - // Recorder.Eventf emits to both core/events and events.k8s.io/events. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - // Cluster-wide secrets+configmaps CRUD: reconciler replicates pull - // secrets and CA bundles from the controller namespace into target - // WAFPolicy namespaces. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - // EnvoyExtensionPolicy CRUD: reconciler renders one EEP per WAF targetRef. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - - ms := rtest.GetResource(resources, kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless") - - // The webhook surface is rendered with the operator CA stamped into the - // ValidatingWebhookConfiguration caBundle. - vwc := rtest.GetResource(resources, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration").(*admissionregistrationv1.ValidatingWebhookConfiguration) - Expect(vwc.Webhooks).To(HaveLen(1)) - Expect(vwc.Webhooks[0].ClientConfig.CABundle).To(Equal([]byte("fake-ca-bundle"))) - }) - - It("should delete the WAF admission webhook surface when the WAF Gateway API add-on is disabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = false - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - toCreate, toDelete := component.Objects() - - // Neither webhook object is created... - Expect(rtest.GetResource(toCreate, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).To(BeNil()) - Expect(rtest.GetResource(toCreate, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).To(BeNil()) - // ...and both are queued for deletion, so disabling the feature (or - // removing the GatewayAPI CR) cleans up an earlier enabled render. - Expect(rtest.GetResource(toDelete, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).NotTo(BeNil()) - Expect(rtest.GetResource(toDelete, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).NotTo(BeNil()) - }) - It("should render all calico kube-controllers resources using CalicoEnterprise on Openshift", func() { expectedResources := []struct { name string @@ -436,23 +264,20 @@ var _ = Describe("kube-controllers rendering tests", func() { version string kind string }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: enterprise.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + {name: enterprise.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: enterprise.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: enterprise.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: enterprise.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: enterprise.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) @@ -465,7 +290,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + dp := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) envs := dp.Spec.Template.Spec.Containers[0].Env @@ -482,8 +307,8 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") + clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ APIGroups: []string{""}, @@ -498,60 +323,7 @@ var _ = Describe("kube-controllers rendering tests", func() { })) }) - It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - // Override configuration to match expected Enterprise config. - instance.Variant = operatorv1.CalicoEnterprise - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", - Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - }) It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise", func() { - var defaultMode int32 = 420 - var kubeControllerTLS certificatemanagement.KeyPairInterface expectedResources := []struct { name string ns string @@ -566,15 +338,14 @@ var _ = Describe("kube-controllers rendering tests", func() { {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } + // The metrics serving TLS (TLS_KEY_PATH/TLS_CRT_PATH/CLIENT_COMMON_NAME env, + // the keypair volume + mount) is layered on by the enterprise modifier, so + // the base render here carries only the trusted bundle. expectedEnv := []corev1.EnvVar{ - {Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, - {Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, - {Name: "CLIENT_COMMON_NAME", Value: "calico-node-prometheus-client-tls"}, {Name: "CA_CRT_PATH", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, } expectedVolumeMounts := []corev1.VolumeMount{ {Name: "tigera-ca-bundle", MountPath: "/etc/pki/tls/certs", ReadOnly: true}, - {Name: "calico-kube-controllers-metrics-tls", MountPath: "/calico-kube-controllers-metrics-tls", ReadOnly: true}, } expectedVolume := []corev1.Volume{ { @@ -585,34 +356,11 @@ var _ = Describe("kube-controllers rendering tests", func() { }, }, }, - { - Name: "calico-kube-controllers-metrics-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "calico-kube-controllers-metrics-tls", - DefaultMode: &defaultMode, - }, - }, - }, } - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair(cli, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - // Override configuration to match expected Enterprise config. instance.Variant = operatorv1.CalicoEnterprise cfg.MetricsPort = 9094 - cfg.MetricsServerTLS = kubeControllerTLS component := kubecontrollers.NewCalicoKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -632,59 +380,15 @@ var _ = Describe("kube-controllers rendering tests", func() { envs := dp.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElements(expectedEnv)) - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElements(expectedVolumeMounts)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Volumes).To(ContainElements(expectedVolume)) Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) }) - It("should mount the WAF admission webhook serving cert and expose its port when WAF is enabled", func() { - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - wafTLS, err := certificateManager.GetOrCreateKeyPair(cli, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = true - cfg.WAFWebhookServerTLS = wafTLS - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - c := dp.Spec.Template.Spec.Containers[0] - - // Serving cert is mounted and advertised to the in-process webhook server. - Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(wafTLS.Volume())) - Expect(c.VolumeMounts).To(ContainElement(wafTLS.VolumeMount(rmeta.OSTypeLinux))) - Expect(c.Env).To(ContainElement(corev1.EnvVar{ - Name: "WAF_WEBHOOK_CERT_DIR", - Value: filepath.Dir(wafTLS.VolumeMountCertificateFilePath()), - })) - - // In-process webhook port exposed for the tigera-waf-webhook Service. - Expect(c.Ports).To(ContainElement(corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: int32(9443), - Protocol: corev1.ProtocolTCP, - })) - - // namespaces patch/update RBAC for the waf-id-range annotation. - clusterRole := rtest.GetResource(resources, "calico-kube-controllers", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - })) - }) - It("should render all es-calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { expectedResources := []struct { name string @@ -693,26 +397,23 @@ var _ = Describe("kube-controllers rendering tests", func() { version string kind string }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: enterprise.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: enterprise.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: enterprise.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + {name: enterprise.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: enterprise.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: enterprise.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } // Override configuration to match expected Enterprise config. instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() Expect(len(resources)).To(Equal(len(expectedResources))) @@ -725,7 +426,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + dp := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) envs := dp.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElement(corev1.EnvVar{ @@ -743,8 +444,8 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(36), "cluster role should have 36 rules") + clusterRole := rtest.GetResource(resources, enterprise.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") Expect(clusterRole.Rules).To(ContainElement( rbacv1.PolicyRule{ APIGroups: []string{""}, @@ -865,7 +566,6 @@ var _ = Describe("kube-controllers rendering tests", func() { It("should add the OIDC prefix env variables", func() { instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 @@ -875,17 +575,17 @@ var _ = Describe("kube-controllers rendering tests", func() { Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, }} - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + depResource := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") Expect(depResource).ToNot(BeNil()) deployment := depResource.(*appsv1.Deployment) var usernamePrefix, groupPrefix string for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { + if container.Name == enterprise.EsKubeController { for _, env := range container.Env { switch env.Name { case "OIDC_AUTH_USERNAME_PREFIX": @@ -1132,20 +832,19 @@ var _ = Describe("kube-controllers rendering tests", func() { When("enableESOIDCWorkaround is true", func() { It("should set the ENABLE_ELASTICSEARCH_OIDC_WORKAROUND env variable to true", func() { instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.ManagementCluster = &operatorv1.ManagementCluster{} cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) resources, _ := component.Objects() - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + depResource := rtest.GetResource(resources, enterprise.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") Expect(depResource).ToNot(BeNil()) deployment := depResource.(*appsv1.Deployment) var esLicenseType string for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { + if container.Name == enterprise.EsKubeController { for _, env := range container.Env { if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { esLicenseType = env.Value @@ -1289,10 +988,9 @@ var _ = Describe("kube-controllers rendering tests", func() { cfg.ManagementClusterConnection = nil } instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + component := enterprise.NewElasticsearchKubeControllers(&cfg) resources, _ := component.Objects() policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) @@ -1306,29 +1004,6 @@ var _ = Describe("kube-controllers rendering tests", func() { ) }) - It("should render init containers when certificate management is enabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.MetricsPort = 9094 - ca, _ := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) - cert, _, _ := ca.Config.GetPEMBytes() // create a valid pem block - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: cert} - - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - tls, err := certificateManager.GetOrCreateKeyPair(cli, kubecontrollers.KubeControllerPrometheusTLSSecret, common.OperatorNamespace(), []string{""}) - Expect(err).NotTo(HaveOccurred()) - - cfg.MetricsServerTLS = tls - - resources, _ := kubecontrollers.NewCalicoKubeControllers(&cfg).Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - csrInitContainer := dp.Spec.Template.Spec.InitContainers[0] - Expect(csrInitContainer.Name).To(Equal(fmt.Sprintf("%v-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) - }) - It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { cfg.K8sServiceEp.Host = "k8shost" cfg.K8sServiceEp.Port = "1234" diff --git a/pkg/render/kubecontrollers/waf_pull_secret.go b/pkg/render/kubecontrollers/waf_pull_secret.go deleted file mode 100644 index 02ada09f8a..0000000000 --- a/pkg/render/kubecontrollers/waf_pull_secret.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubecontrollers - -import ( - "encoding/json" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/tigera/operator/pkg/common" -) - -// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret -// (tigera-waf-pull-secret) by merging the registry auths of every Installation -// pull secret. The EnvoyExtensionPolicy image source takes a single -// pullSecretRef, so a merged secret is the only way to honor multiple -// Installation pull secrets for the Coraza wasm OCI pull (e.g. the Tigera pull -// secret plus credentials for a private registry mirror). -// -// If the same registry appears in more than one secret, the first secret in -// Installation order wins. Secrets that cannot be parsed are skipped and their -// names returned, so the caller can log them without failing the reconcile. -// Returns a nil Secret when no registry auths could be collected. -func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { - merged := map[string]json.RawMessage{} - var skipped []string - for _, s := range pullSecrets { - auths, err := registryAuths(s) - if err != nil { - skipped = append(skipped, s.Name) - continue - } - for registry, auth := range auths { - if _, ok := merged[registry]; !ok { - merged[registry] = auth - } - } - } - if len(merged) == 0 { - return nil, skipped - } - - // Marshalling a map sorts its keys, so the rendered bytes are deterministic - // and do not churn the object on every reconcile. - data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) - if err != nil { - // Each auth entry round-trips from a successful Unmarshal above, so - // this cannot fail in practice; treat it as nothing to render. - return nil, skipped - } - - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, - Type: corev1.SecretTypeDockerConfigJson, - Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, - }, skipped -} - -// registryAuths extracts the per-registry auth entries from a pull secret of -// either the dockerconfigjson type (auths nested under an "auths" key) or the -// legacy dockercfg type (a bare registry -> auth map). -func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { - if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { - var cfg struct { - Auths map[string]json.RawMessage `json:"auths"` - } - if err := json.Unmarshal(raw, &cfg); err != nil { - return nil, err - } - if len(cfg.Auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return cfg.Auths, nil - } - if raw, ok := s.Data[corev1.DockerConfigKey]; ok { - var auths map[string]json.RawMessage - if err := json.Unmarshal(raw, &auths); err != nil { - return nil, err - } - if len(auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return auths, nil - } - return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) -} diff --git a/pkg/render/logstorage/esgateway/esgateway_test.go b/pkg/render/logstorage/esgateway/esgateway_test.go index 7f29d298d7..5a76cc2c34 100644 --- a/pkg/render/logstorage/esgateway/esgateway_test.go +++ b/pkg/render/logstorage/esgateway/esgateway_test.go @@ -37,12 +37,12 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/podaffinity" rtest "github.com/tigera/operator/pkg/render/common/test" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -81,9 +81,9 @@ var _ = Describe("ES Gateway rendering tests", func() { ESGatewayKeyPair: kp, TrustedBundle: bundle, KubeControllersUserSecrets: []*corev1.Secret{ - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, }, ClusterDomain: clusterDomain, EsAdminUserName: "elastic", @@ -95,9 +95,9 @@ var _ = Describe("ES Gateway rendering tests", func() { It("should render an ES Gateway deployment and all supporting resources", func() { expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, @@ -134,9 +134,9 @@ var _ = Describe("ES Gateway rendering tests", func() { installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: secret.Data[corev1.TLSCertKey]} expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: enterprise.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, diff --git a/pkg/render/manager.go b/pkg/render/manager.go index 765918beb3..f002cb1aff 100644 --- a/pkg/render/manager.go +++ b/pkg/render/manager.go @@ -262,10 +262,10 @@ func (c *managerComponent) Objects() ([]client.Object, []client.Object) { // For multi-tenant environments, the management cluster itself isn't shown in the UI so we only need to create these // when there is no tenant. objsToCreate = append(objsToCreate, - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), + ManagerClusterWideSettingsGroup(), + ManagerUserSpecificSettingsGroup(), + ManagerClusterWideTigeraLayer(), + ManagerClusterWideDefaultView(), ) // Continue to create the legacy namespace so that we can create our external name service that points to the new // manager service. This will help ease transition for customers and avoid outages caused by the name and namespace @@ -1358,10 +1358,10 @@ func (c *managerComponent) multiTenantManagedClustersAccess() []client.Object { return objects } -// managerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" +// ManagerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" // // Calico Enterprise only -func managerClusterWideSettingsGroup() *v3.UISettingsGroup { +func ManagerClusterWideSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1373,10 +1373,10 @@ func managerClusterWideSettingsGroup() *v3.UISettingsGroup { } } -// managerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" +// ManagerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" // // Calico Enterprise only -func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { +func ManagerUserSpecificSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1389,11 +1389,11 @@ func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { } } -// managerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains +// ManagerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains // all of the tigera namespaces. // // Calico Enterprise only -func managerClusterWideTigeraLayer() *v3.UISettings { +func ManagerClusterWideTigeraLayer() *v3.UISettings { namespaces := []string{ "tigera-compliance", "tigera-dex", @@ -1439,11 +1439,11 @@ func managerClusterWideTigeraLayer() *v3.UISettings { } } -// managerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows +// ManagerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows // everything and uses the tigera-infrastructure layer. // // Calico Enterprise only -func managerClusterWideDefaultView() *v3.UISettings { +func ManagerClusterWideDefaultView() *v3.UISettings { return &v3.UISettings{ TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/render/node.go b/pkg/render/node.go index db6d5ae4a3..9a838cf5cc 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -35,6 +35,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" "github.com/tigera/operator/pkg/render/common/configmap" rmeta "github.com/tigera/operator/pkg/render/common/meta" @@ -72,9 +73,9 @@ const ( ) var ( - // The port used by calico/node to report Calico Enterprise BGP metrics. + // NodeBGPReporterPort is the port used by calico/node to report Calico Enterprise BGP metrics. // This is currently not intended to be user configurable. - nodeBGPReporterPort int32 = 9900 + NodeBGPReporterPort int32 = 9900 NodeTLSSecretName = "node-certs" NodeTLSSecretNameNonClusterHost = NodeTLSSecretName + TyphaNonClusterHostSuffix @@ -115,11 +116,9 @@ type NodeConfiguration struct { GoldmaneIP string // Optional fields. - LogCollector *operatorv1.LogCollector - MigrateNamespaces bool - NodeAppArmorProfile string - BirdTemplates map[string]string - NodeReporterMetricsPort int + MigrateNamespaces bool + NodeAppArmorProfile string + BirdTemplates map[string]string // CanRemoveCNIFinalizer specifies whether CNI plugin is still needed during uninstall since the CNI plugin and // associated RBAC resources are required for pod teardown to succeed. Setting this to true removes @@ -127,8 +126,6 @@ type NodeConfiguration struct { // For details on why this is needed see 'Node and Installation finalizer' in the core_controller. CanRemoveCNIFinalizer bool - PrometheusServerTLS certificatemanagement.KeyPairInterface - // BGPLayouts is returned by the rendering code after modifying its namespace // so that it can be deployed into the cluster. // TODO: The controller should pass the contents, the renderer should build its own @@ -146,11 +143,12 @@ type NodeConfiguration struct { // should this value change. BindMode string - FelixPrometheusMetricsEnabled bool - - FelixPrometheusMetricsPort int - V3CRDs bool + + // ImageOverrides lets a variant swap the node and cni-plugins images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } // Node creates the node daemonset and other resources for the daemonset to operate normally. @@ -185,18 +183,11 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { } c.calicoImage = appendIfErr(components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is)) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameNode, components.ComponentCalicoNode, c.cfg.Installation) + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if c.installUpstreamPlugins() { - if c.cfg.Installation.Variant.IsEnterprise() { - c.cniPluginsImage = appendIfErr(components.GetReference(components.ComponentTigeraCNIPlugins, reg, path, prefix, is)) - } else { - c.cniPluginsImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIPlugins, reg, path, prefix, is)) - } - } - switch { - case c.cfg.Installation.Variant.IsEnterprise(): - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNode, reg, path, prefix, is)) - default: - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNode, reg, path, prefix, is)) + cniPluginsImage := c.cfg.ImageOverrides.Resolve(ComponentNameCNIPlugins, components.ComponentCalicoCNIPlugins, c.cfg.Installation) + c.cniPluginsImage = appendIfErr(components.GetReference(cniPluginsImage, reg, path, prefix, is)) } if len(errMsgs) != 0 { @@ -209,6 +200,10 @@ func (c *nodeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *nodeComponent) ModifierKey() string { + return ComponentNameNode +} + func (c *nodeComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ c.nodeServiceAccount(), @@ -234,11 +229,6 @@ func (c *nodeComponent) Objects() ([]client.Object, []client.Object) { var objsToDelete []client.Object - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.nodeCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -566,34 +556,6 @@ func (c *nodeComponent) nodeRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Calico Enterprise needs to be able to read additional resources. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Tigera Secure updates status for packet captures. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "packetcaptures", - "packetcaptures/status", - }, - Verbs: []string{"update"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -655,14 +617,6 @@ func (c *nodeComponent) cniPluginRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // The Network resource is only available in Enterprise / Cloud at this time. - role.Rules = append(role.Rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get"}, - }) - } return role } @@ -950,18 +904,11 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo if len(c.cfg.BirdTemplates) != 0 { annotations[birdTemplateHashAnnotation] = rmeta.AnnotationHash(c.cfg.BirdTemplates) } - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if c.cfg.TLS.NodeSecret.UseCertificateManagement() { initContainers = append(initContainers, c.cfg.TLS.NodeSecret.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) } - if c.cfg.PrometheusServerTLS != nil && c.cfg.PrometheusServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.PrometheusServerTLS.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) - } - if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) } @@ -1077,10 +1024,6 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, c.cniContainer()) } - if c.collectProcessPathEnabled() { - ds.Spec.Template.Spec.HostPID = true - } - setNodeCriticalPod(&(ds.Spec.Template)) if c.cfg.MigrateNamespaces { migration.LimitDaemonSetToMigratedNodes(&ds) @@ -1111,13 +1054,16 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { c.cfg.TLS.TrustedBundle.Volume(), c.cfg.TLS.NodeSecret.Volume(), c.varRunCalicoVolume(), - corev1.Volume{Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + // The Calico log directory. The CNI plugin logs to the cni/ subdirectory of + // this, and Felix writes its flow/DNS logs here on the enterprise variant. + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, // Volume for the containing directory so that the init container can mount the child bpf directory if needed. - corev1.Volume{Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, + {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, // Volume for the bpffs itself, used by the main node container. - corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, + {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, // Volume used by mount-cgroupv2 init container to access root cgroup name space of node. - corev1.Volume{Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, + {Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, } if c.vppDataplaneEnabled() { @@ -1131,7 +1077,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { volumes = append(volumes, corev1.Volume{Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.BinDir, Type: &dirOrCreate}}}) volumes = append(volumes, corev1.Volume{Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.ConfDir}}}) - volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}) } if c.installUpstreamPlugins() { // Staging volume populated by the cni-plugins init container and read @@ -1139,16 +1084,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) } - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) - } - // Create and append flexvolume if c.cfg.Installation.FlexVolumePath != "None" { volumes = append(volumes, corev1.Volume{ @@ -1185,10 +1120,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { }, }) } - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -1208,12 +1139,6 @@ func (c *nodeComponent) vppDataplaneEnabled() bool { *c.cfg.Installation.CalicoNetwork.LinuxDataplane == operatorv1.LinuxDataplaneVPP } -func (c *nodeComponent) collectProcessPathEnabled() bool { - return c.cfg.LogCollector != nil && - c.cfg.LogCollector.Spec.CollectProcessPath != nil && - *c.cfg.LogCollector.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable -} - // cniContainer creates the node's init container that installs CNI. func (c *nodeComponent) cniContainer() corev1.Container { // Determine environment to pass to the CNI init container. @@ -1382,12 +1307,6 @@ func (c *nodeComponent) cniEnvvars() []corev1.EnvVar { envVars = append(envVars, c.cfg.K8sServiceEp.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - envVars = append(envVars, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - } - return envVars } @@ -1431,15 +1350,9 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { if c.vppDataplaneEnabled() { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/usr/local/bin/felix-plugins", Name: "felix-plugins", ReadOnly: true}) } - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - nodeVolumeMounts = append(nodeVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { - cniLogMount := corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false} - nodeVolumeMounts = append(nodeVolumeMounts, cniLogMount) - } + // Mount the Calico log directory. The CNI plugin writes to the cni/ subdirectory + // and, on the enterprise variant, Felix writes its flow/DNS logs here too. + nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) @@ -1475,9 +1388,6 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { SubPath: BGPLayoutConfigMapKey, }) } - if c.cfg.PrometheusServerTLS != nil { - nodeVolumeMounts = append(nodeVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return nodeVolumeMounts } @@ -1588,10 +1498,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { } } - if c.collectProcessPathEnabled() { - nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) - } - // Determine MTU to use. If specified explicitly, use that. Otherwise, set defaults based on an overall // MTU of 1460. mtu := getMTU(c.cfg.Installation) @@ -1685,35 +1591,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - extraNodeEnv = append(extraNodeEnv, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - nodeEnv = append(nodeEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -1792,10 +1669,7 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr var readinessCmd []string readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--bird-ready", "--felix-ready"} - if c.cfg.Installation.Variant.IsEnterprise() { - readinessCmd = append(readinessCmd, "--bgp-metrics-ready") - } - // If not using BGP or using VPP, don't check bird status (or bgp metrics server for enterprise). + // If not using BGP or using VPP, don't check bird status. if !bgpEnabled(c.cfg.Installation) || c.vppDataplaneEnabled() { readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--felix-ready"} } @@ -1821,56 +1695,6 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr return lp, rp } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *nodeComponent) nodeMetricsService() *corev1.Service { - ports := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - } - - if c.cfg.FelixPrometheusMetricsEnabled { - felixMetricsPort := int32(c.cfg.FelixPrometheusMetricsPort) - - ports = append(ports, corev1.ServicePort{ - Name: "felix-metrics-port", - Port: felixMetricsPort, - TargetPort: intstr.FromInt(int(felixMetricsPort)), - Protocol: corev1.ProtocolTCP, - }) - } - - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: CalicoNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": CalicoNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": CalicoNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: ports, - }, - } -} - // getAutodetectionMethod returns the IP auto detection method in a form understandable by the calico/node // startup processing. It returns an empty string if IP auto detection should not be enabled. func getAutodetectionMethod(ad *operatorv1.NodeAddressAutodetection) string { diff --git a/pkg/render/node_enterprise_test.go b/pkg/render/node_enterprise_test.go new file mode 100644 index 0000000000..7696646454 --- /dev/null +++ b/pkg/render/node_enterprise_test.go @@ -0,0 +1,218 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// These tests run the real node/typha render output through the registered +// enterprise modifiers. The render suite registers the enterprise extensions in +// its BeforeSuite, so this exercises the same integrated behavior the operator +// binary produces - and, importantly, catches a modifier whose FindObject stops +// matching because render renamed an object or container. +var _ = Describe("node enterprise modifier integration", func() { + var ( + cli client.Client + certManager certificatemanager.CertificateManager + typhaNodeTLS *render.TyphaNodeTLS + instance *operatorv1.InstallationSpec + renderCtx extensions.RenderContext + nodePrometheusTLS certificatemanagement.KeyPairInterface + ) + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + BeforeEach(func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + var err error + certManager, err = certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS = getTyphaNodeTLS(cli, certManager) + + nodePrometheusTLS, err = certManager.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics"}) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) + + confDir, binDir := render.DefaultCNIDirectories(operatorv1.ProviderNone) + bgp := operatorv1.BGPEnabled + instance = &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{ + Type: operatorv1.PluginCalico, + IPAM: &operatorv1.IPAMSpec{Type: operatorv1.IPAMPluginCalico}, + BinDir: &binDir, + ConfDir: &confDir, + }, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + BGP: &bgp, + IPPools: []operatorv1.IPPool{{CIDR: "192.168.1.0/16"}}, + }, + } + + // Build the render context the way the controller does: run the enterprise + // controller extension, which stashes the node prometheus keypair in the + // context for the node modifier to read. + cc := extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: cli, + CertificateManager: certManager, + } + renderCtx, _, err = ext.ExtendContext(cc) + Expect(err).NotTo(HaveOccurred()) + }) + + // renderNodeObjects renders the real node component and applies the registered + // modifier, exactly as the componentHandler does. + renderNodeObjects := func(rc extensions.RenderContext) []client.Object { + cfg := &render.NodeConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + IPPools: instance.CalicoNetwork.IPPools, + } + comp := render.Node(cfg) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + out, _ := applyExtensions(ext, render.ComponentNameNode, rc, objs, nil) + return out + } + + It("appends the node metrics service to the real render output", func() { + objs := renderNodeObjects(renderCtx) + svc, ok := extensions.FindObject[*corev1.Service](objs, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue(), "expected the modifier to append %s", render.CalicoNodeMetricsService) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + }) + + It("adds the enterprise rules to the real cluster roles", func() { + objs := renderNodeObjects(renderCtx) + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("networks")))) + }) + + It("rewrites the real node daemonset for enterprise", func() { + objs := renderNodeObjects(renderCtx) + ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + + c := nodeContainer(ds) + Expect(c).NotTo(BeNil()) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + )) + // The reporter cert env is wired from the NodePrometheusTLS keypair the + // builder creates, and the modifier mounts that keypair onto the daemonset. + Expect(c.Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(nodePrometheusTLS.Volume())) + Expect(c.VolumeMounts).To(ContainElement(nodePrometheusTLS.VolumeMount(rmeta.OSTypeLinux))) + Expect(ds.Spec.Template.Annotations).To(HaveKey(nodePrometheusTLS.HashAnnotationKey())) + + // BGP is enabled, so the bird readiness check is present and the modifier + // adds the BGP metrics check. + Expect(c.ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) + }) + + It("enables process-path collection when the LogCollector requests it", func() { + enable := operatorv1.CollectProcessPathEnable + Expect(cli.Create(context.Background(), &operatorv1.LogCollector{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + Spec: operatorv1.LogCollectorSpec{CollectProcessPath: &enable}, + })).NotTo(HaveOccurred()) + + rc, _, err := ext.ExtendContext(extensions.ControllerContext{ + RenderContext: extensions.RenderContext{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Controller: extensions.InstallationController, + Ctx: context.Background(), + Client: cli, + CertificateManager: certManager, + }) + Expect(err).NotTo(HaveOccurred()) + + ds, ok := extensions.FindObject[*appsv1.DaemonSet](renderNodeObjects(rc), common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + Expect(ds.Spec.Template.Spec.HostPID).To(BeTrue()) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"})) + }) + + It("adds the enterprise rules to the real typha cluster role", func() { + comp := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + }) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + objs, _ = applyExtensions(ext, render.ComponentNameTypha, renderCtx, objs, nil) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index 5c5c0f50eb..b052be0ef4 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -134,14 +134,13 @@ var _ = Describe("Node rendering tests", func() { // Create a default configuration. cfg = render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: defaultInstance, - TLS: typhaNodeTLS, - ClusterDomain: defaultClusterDomain, - FelixHealthPort: 9099, - IPPools: defaultInstance.CalicoNetwork.IPPools, - FelixPrometheusMetricsEnabled: false, - FelixPrometheusMetricsPort: 9098, + K8sServiceEp: k8sServiceEp, + Installation: defaultInstance, + TLS: typhaNodeTLS, + ClusterDomain: defaultClusterDomain, + FelixHealthPort: 9099, + IPPools: defaultInstance.CalicoNetwork.IPPools, + ImageOverrides: ext.Images(), } }) @@ -321,7 +320,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -360,7 +359,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -368,7 +367,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render node correctly for BPF dataplane", func() { @@ -513,7 +512,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -552,7 +551,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -560,7 +559,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render an explicitly configured MTU", func() { @@ -639,143 +638,6 @@ var _ = Describe("Node rendering tests", func() { } }) - It("should render all resources for a default configuration using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9099"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - // Expect 2 Ports when FelixPrometheusMetricsEnabled is false - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(len(ms.Spec.Ports)).To(Equal(2)) - - dirMustExist := corev1.HostPathDirectory - bpfVol := corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}} - Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(bpfVol)) - - bpfVolMount := corev1.VolumeMount{MountPath: "/sys/fs/bpf", Name: "bpffs"} - Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(bpfVolMount)) - - verifyProbesAndLifecycle(ds, false, true) - }) - - It("should render felix service metric with FelixPrometheusMetricPort when FelixPrometheusMetricsEnabled is true", func() { - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixPrometheusMetricsEnabled = true - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - expectedServicePorts := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: 9900, - TargetPort: intstr.FromInt(int(9900)), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "felix-metrics-port", - Port: 9098, - TargetPort: intstr.FromInt(int(9098)), - Protocol: corev1.ProtocolTCP, - }, - } - - // Expect 3 Ports when FelixPrometheusMetricsEnabled is true - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.Ports).To(Equal(expectedServicePorts)) - }) - It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { name string @@ -913,7 +775,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -952,7 +814,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -962,7 +824,7 @@ var _ = Describe("Node rendering tests", func() { // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1051,6 +913,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1085,6 +948,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1096,7 +960,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should return customized CNI directories when specified", func() { @@ -1116,7 +980,7 @@ var _ = Describe("Node rendering tests", func() { expectedVols := []corev1.Volume{ {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ContainElements(expectedVols)) @@ -1165,7 +1029,7 @@ var _ = Describe("Node rendering tests", func() { } // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }, Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal, []corev1.EnvVar{ {Name: "FELIX_INTERFACEPREFIX", Value: "gke"}, @@ -1317,7 +1181,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1356,7 +1220,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, } Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) @@ -1365,7 +1229,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1452,6 +1316,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1486,6 +1351,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1497,7 +1363,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render all resources when running on openshift", func() { @@ -1564,7 +1430,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/multus/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "flexvol-driver-host", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds", Type: &dirOrCreate}}}, @@ -1628,215 +1494,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - verifyProbesAndLifecycle(ds, true, false) - }) - - It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // calico-node clusterRole should have openshift securitycontextconstraints PolicyRule - nodeRole := rtest.GetResource(resources, "calico-node", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(nodeRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"privileged"}, - })) - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) - }) - - It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - - // The RKE2 envvar overrides. - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) - - // The metrics service should have the correct configuration. - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") + verifyProbesAndLifecycle(ds, true) }) It("should render volumes and node volumemounts when bird templates are provided", func() { @@ -2095,12 +1753,11 @@ var _ = Describe("Node rendering tests", func() { It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) @@ -2109,7 +1766,8 @@ var _ = Describe("Node rendering tests", func() { ds := dsResource.(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) - // It should have the reporter port, though. + // The reporter port env is added by the enterprise node modifier, not the + // base render, so it should be absent here. expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) }) @@ -2121,7 +1779,7 @@ var _ = Describe("Node rendering tests", func() { component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) @@ -2949,7 +2607,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) DescribeTable("test node probes", @@ -2974,7 +2632,7 @@ var _ = Describe("Node rendering tests", func() { Expect(dsResource).ToNot(BeNil()) ds := dsResource.(*appsv1.DaemonSet) - verifyProbesAndLifecycle(ds, isOpenshift, isEnterprise) + verifyProbesAndLifecycle(ds, isOpenshift) }, Entry("k8s Calico OS no BGP", false, false, operatorv1.BGPDisabled), @@ -3293,7 +2951,7 @@ var _ = Describe("Node rendering tests", func() { }) // verifyProbesAndLifecycle asserts the expected node liveness and readiness probe plus pod lifecycle settings. -func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bool) { +func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift bool) { // Verify readiness and liveness probes. expectedReadiness := &corev1.Probe{ PeriodSeconds: 10, @@ -3327,14 +2985,14 @@ func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bo } ExpectWithOffset(1, found).To(BeTrue()) + // The base render produces the same readiness command for all variants; the + // enterprise --bgp-metrics-ready check is added by the node modifier and is + // covered in the enterprise package tests. var expectedReadinessCmd []string - switch { - case !bgp: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} - case bgp && isEnterprise: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready", "--bgp-metrics-ready"} - case bgp: + if bgp { expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready"} + } else { + expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} } expectedReadiness.ProbeHandler = corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: expectedReadinessCmd}} diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 2b97cfaeb9..beef705146 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -66,9 +66,7 @@ func allCalicoComponents( nodeAppArmorProfile string, clusterDomain string, kubeControllersMetricsPort int, - nodeReporterMetricsPort int, bgpLayout *corev1.ConfigMap, - logCollector *operatorv1.LogCollector, ) ([]render.Component, error) { namespaces := render.Namespaces(&render.NamespaceConfiguration{Installation: cr, PullSecrets: pullSecrets}) @@ -79,17 +77,15 @@ func allCalicoComponents( secretsAndConfigMaps := render.NewCreationPassthrough(objs...) nodeCfg := &render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: cr, - TLS: typhaNodeTLS, - NodeAppArmorProfile: nodeAppArmorProfile, - ClusterDomain: clusterDomain, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - LogCollector: logCollector, - BirdTemplates: bt, - MigrateNamespaces: up, - FelixHealthPort: 9099, + K8sServiceEp: k8sServiceEp, + Installation: cr, + TLS: typhaNodeTLS, + NodeAppArmorProfile: nodeAppArmorProfile, + ClusterDomain: clusterDomain, + BGPLayouts: bgpLayout, + BirdTemplates: bt, + MigrateNamespaces: up, + FelixHealthPort: 9099, } typhaCfg := &render.TyphaConfiguration{ K8sServiceEp: k8sServiceEp, @@ -111,13 +107,12 @@ func allCalicoComponents( } winCfg := &render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{}, - Installation: cr, - ClusterDomain: clusterDomain, - TLS: typhaNodeTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{}, + Installation: cr, + ClusterDomain: clusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, } nodeCertComponent := rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{ @@ -219,22 +214,22 @@ var _ = Describe("Rendering tests", func() { // - 6 kube-controllers resources (ServiceAccount, ClusterRole, Binding, Deployment, Service, Secret,RoleBinding) // - 1 namespace // - 2 Windows node resources (ConfigMap, DaemonSet) - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) It("should render all resources when variant is Tigera Secure", func() { - // For this scenario, we expect the basic resources plus the following for Tigera Secure: - // - X Same as default config - // - 1 Service to expose calico/node metrics. - // - 1 Service to expose Windows calico/node metrics. + // For this scenario, we expect the basic resources plus the following for Tigera Secure. + // The calico/node and Windows calico/node metrics Services are added by the + // enterprise modifiers at the componentHandler, not by Objects(), so they do + // not appear in this render-only aggregation. var nodeMetricsPort int32 = 9081 instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - Expect(componentCount(c)).To(Equal((5 + 3 + 4 + 1 + 6 + 6 + 1 + 2) + 1 + 1)) + Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) It("should render all resources when variant is Tigera Secure and Management Cluster", func() { @@ -245,7 +240,7 @@ var _ = Describe("Rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) expectedResources := []client.Object{ @@ -267,7 +262,6 @@ var _ = Describe("Rendering tests", func() { &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-node-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, @@ -279,8 +273,8 @@ var _ = Describe("Rendering tests", func() { &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllersDeploymentName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-kube-controllers-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - // Windows node objects. - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.WindowsNodeMetricsService, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + // Windows node objects. The Windows node-metrics Service is added by the + // enterprise modifier at the componentHandler, so it is not in this output. &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config-windows", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, @@ -307,7 +301,7 @@ var _ = Describe("Rendering tests", func() { It("should render calico with a apparmor profile if annotation is present in installation", func() { apparmorProf := "foobar" - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -331,7 +325,7 @@ var _ = Describe("Rendering tests", func() { } bgpLayout.Name = "bgp-layout" bgpLayout.Namespace = common.OperatorNamespace() - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, bgpLayout, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, bgpLayout) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cm *corev1.ConfigMap var ds *appsv1.DaemonSet @@ -352,42 +346,8 @@ var _ = Describe("Rendering tests", func() { Expect(ds.Spec.Template.Annotations["hash.operator.tigera.io/bgp-layout"]).NotTo(BeEmpty()) }) - It("should handle collectProcessPath in logCollector", func() { - testNode := func(processPath operatorv1.CollectProcessPathOption, expectedHostPID bool) { - var logCollector operatorv1.LogCollector - logCollector.Spec.CollectProcessPath = &processPath - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, &logCollector) - Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - var ds *appsv1.DaemonSet - for _, comp := range comps { - resources, _ := comp.Objects() - r := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") - if r != nil { - ds = r.(*appsv1.DaemonSet) - } - } - checkEnvVar := func(ds *appsv1.DaemonSet) bool { - envPresent := false - for _, env := range ds.Spec.Template.Spec.Containers[0].Env { - if env.Name == "FELIX_FLOWLOGSCOLLECTPROCESSPATH" { - envPresent = true - if env.Value == "true" { - return true - } - } - } - return !envPresent - } - Expect(ds).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.HostPID).To(Equal(expectedHostPID)) - Expect(checkEnvVar(ds)).To(Equal(true)) - } - testNode(operatorv1.CollectProcessPathEnable, true) - testNode(operatorv1.CollectProcessPathDisable, false) - }) - It("should set node priority class to system-node-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -403,7 +363,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set typha priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { @@ -419,7 +379,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set kube controllers priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { diff --git a/pkg/render/typha.go b/pkg/render/typha.go index 861dd0b324..3632978f81 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -113,6 +113,8 @@ func (c *typhaComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } +func (c *typhaComponent) ModifierKey() string { return ComponentNameTypha } + func (c *typhaComponent) Objects() ([]client.Object, []client.Object) { pdb := c.typhaPodDisruptionBudget() if overrides := c.cfg.Installation.TyphaPodDisruptionBudget; overrides != nil { @@ -355,26 +357,6 @@ func (c *typhaComponent) typhaRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Tigera Secure needs to be able to read licenses, and config. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "deeppacketinspections", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -632,15 +614,6 @@ func (c *typhaComponent) typhaEnvVars(typhaSecret certificatemanagement.KeyPairI typhaEnv = append(typhaEnv, corev1.EnvVar{Name: "FELIX_INTERFACEPREFIX", Value: "azv"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - typhaEnv = append(typhaEnv, corev1.EnvVar{ - Name: "MULTI_INTERFACE_MODE", - Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value(), - }) - } - } - // If host-local IPAM is in use, we need to configure typha to use the Kubernetes pod CIDR. cni := c.cfg.Installation.CNI if cni != nil && cni.IPAM != nil && cni.IPAM.Type == operatorv1.IPAMPluginHostLocal { diff --git a/pkg/render/windows.go b/pkg/render/windows.go index 28df89c78d..dd9692c77f 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -24,17 +24,16 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) const ( @@ -49,14 +48,17 @@ func Windows( } type WindowsConfiguration struct { - K8sServiceEp k8sapi.ServiceEndpoint - K8sDNSServers []string - Installation *operatorv1.InstallationSpec - ClusterDomain string - TLS *TyphaNodeTLS - PrometheusServerTLS certificatemanagement.KeyPairInterface - NodeReporterMetricsPort int - VXLANVNI int + K8sServiceEp k8sapi.ServiceEndpoint + K8sDNSServers []string + Installation *operatorv1.InstallationSpec + ClusterDomain string + TLS *TyphaNodeTLS + VXLANVNI int + + // ImageOverrides lets a variant swap the windows node and CNI images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } type windowsComponent struct { @@ -77,13 +79,10 @@ func (c *windowsComponent) ResolveImages(is *operatorv1.ImageSet) error { return imageName } - if c.cfg.Installation.Variant.IsEnterprise() { - c.cniImage = appendIfErr(components.GetReference(components.ComponentTigeraCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNodeWindows, reg, path, prefix, is)) - } else { - c.cniImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNodeWindows, reg, path, prefix, is)) - } + cniImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, c.cfg.Installation) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, c.cfg.Installation) + c.cniImage = appendIfErr(components.GetReference(cniImage, reg, path, prefix, is)) + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if len(errMsgs) != 0 { return fmt.Errorf("%s", strings.Join(errMsgs, ",")) @@ -95,6 +94,8 @@ func (c *windowsComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeWindows } +func (c *windowsComponent) ModifierKey() string { return ComponentNameWindows } + func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { // Clean up old windows upgrader daemonset if present objsToDelete := []client.Object{ @@ -116,11 +117,6 @@ func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{} - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.windowsCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -135,43 +131,6 @@ func (c *windowsComponent) Ready() bool { return true } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *windowsComponent) nodeMetricsService() *corev1.Service { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: WindowsNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": WindowsNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": WindowsNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - }, - }, - } -} - // windowsCNIConfigMap returns a config map containing the CNI network config to be installed on each node. // Returns nil if no configmap is needed. func (c *windowsComponent) windowsCNIConfigMap() *corev1.ConfigMap { @@ -380,8 +339,8 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, c.cfg.TLS.TrustedBundle.Volume(), c.cfg.TLS.NodeSecret.Volume(), - corev1.Volume{Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - corev1.Volume{Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, } // If needed for this configuration, then include the CNI volumes. @@ -392,20 +351,6 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: c.cfg.Installation.WindowsNodes.CNILogDir, Type: &dirOrCreate}}}) } - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) - } - - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -483,7 +428,6 @@ func (c *windowsComponent) nodeContainer() corev1.Container { // felixContainer creates the windows felix container. func (c *windowsComponent) felixContainer() corev1.Container { - lp, rp := c.windowsLivenessReadinessProbes() return corev1.Container{ @@ -663,31 +607,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - windowsEnv = append(windowsEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -698,20 +617,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, extraNodeEnv...) } - // Configure provider specific environment variables here. - switch c.cfg.Installation.KubernetesProvider { - case operatorv1.ProviderOpenShift: - if c.cfg.Installation.Variant.IsEnterprise() { - // We need to configure a non-default trusted DNS server, since there's no kube-dns. - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) - } - case operatorv1.ProviderRKE2: - // For RKE2, configure a non-default trusted DNS server, as the DNS service is not named "kube-dns". - if c.cfg.Installation.Variant.IsEnterprise() { - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) - } - } - if c.cfg.Installation.CNI.Type != operatorv1.PluginCalico { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}) } @@ -730,12 +635,7 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { corev1.VolumeMount{MountPath: "/var/run/calico", Name: "var-run-calico"}, corev1.VolumeMount{MountPath: "/var/lib/calico", Name: "var-lib-calico"}) - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - windowsVolumeMounts = append(windowsVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { + if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}) } @@ -743,9 +643,6 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) } - if c.cfg.PrometheusServerTLS != nil { - windowsVolumeMounts = append(windowsVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return windowsVolumeMounts } @@ -801,9 +698,6 @@ func (c *windowsComponent) windowsDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1 initContainers := []corev1.Container{c.uninstallContainer()} annotations := c.cfg.TLS.TrustedBundle.HashAnnotations() - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index f0afe2007c..1d92427a6b 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -35,12 +35,26 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) +// renderWindows renders the windows component and applies the registered +// enterprise modifier the way the componentHandler does, so enterprise tests +// exercise the integrated output (image overrides come from ResolveImages; the +// metrics service, env, volumes and mounts come from the modifier). +func renderWindows(cfg *render.WindowsConfiguration) []client.Object { + comp := render.Windows(cfg) + ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) + objs, _ := comp.Objects() + rc := extensions.RenderContext{Installation: cfg.Installation} + out, _ := applyExtensions(ext, render.ComponentNameWindows, rc, objs, nil) + return out +} + var _ = Describe("Windows rendering tests", func() { var defaultInstance *operatorv1.InstallationSpec var typhaNodeTLS *render.TyphaNodeTLS @@ -107,12 +121,13 @@ var _ = Describe("Windows rendering tests", func() { // Create a default configuration. cfg = render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{"10.96.0.10"}, - Installation: defaultInstance, - ClusterDomain: defaultClusterDomain, - TLS: typhaNodeTLS, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{"10.96.0.10"}, + Installation: defaultInstance, + ClusterDomain: defaultClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, + ImageOverrides: ext.Images(), } }) @@ -694,16 +709,13 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. @@ -1686,18 +1698,15 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. @@ -1842,18 +1851,15 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, } defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) // Should render the correct resources. @@ -2136,11 +2142,8 @@ var _ = Describe("Windows rendering tests", func() { It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") @@ -2159,9 +2162,7 @@ var _ = Describe("Windows rendering tests", func() { var nodeMetricsPort int32 = 1234 defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = &nodeMetricsPort - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources := renderWindows(&cfg) Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")