From e3b08493afe918e3f31cda18e03acd90ccf948d3 Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Sun, 1 Nov 2026 14:57:09 +0100 Subject: [PATCH] feat: Separate rate limits for management and managed clusters --kube-api-qps and --kube-api-burst now default to 40 and 120 and apply to the client for the management cluster. Add --managed-cluster-kube-api-qps and --managed-cluster-kube-api-burst (default 20 and 60) for the clients to managed clusters, passed to libsveltos with clusterproxy.SetRestConfigLimits. Managed cluster clients were limited by the client-go defaults (5 QPS, burst 10). Requires a libsveltos release containing SetRestConfigLimits. --- go.mod | 2 +- go.sum | 4 ++-- main.go | 39 ++++++++++++++++++++++++++++++--------- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 943c151..612e8b5 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/onsi/ginkgo/v2 v2.33.1 github.com/onsi/gomega v1.44.0 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v1.17.0 + github.com/projectsveltos/libsveltos v1.17.1-0.20261101135809-65798348bae6 github.com/prometheus/client_golang v1.25.0 github.com/spf13/pflag v1.0.10 github.com/yuin/gopher-lua v1.1.2 diff --git a/go.sum b/go.sum index cda48f9..127fbcd 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectsveltos/libsveltos v1.17.0 h1:7oIvY+bg9oLRYm8wlyZypYVvCcMspYl/FT9VG8XvQG0= -github.com/projectsveltos/libsveltos v1.17.0/go.mod h1:dHJrl1V5/WrNda5wzxbHs2dlsvoV/FUlwK1TXy2X3pA= +github.com/projectsveltos/libsveltos v1.17.1-0.20261101135809-65798348bae6 h1:lwfrK/ZH9Ee2YAcEdbS0T18QMp/ea7BvEt967oKQnu4= +github.com/projectsveltos/libsveltos v1.17.1-0.20261101135809-65798348bae6/go.mod h1:dHJrl1V5/WrNda5wzxbHs2dlsvoV/FUlwK1TXy2X3pA= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY= github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos= diff --git a/main.go b/main.go index b723918..6393291 100644 --- a/main.go +++ b/main.go @@ -38,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" cliflag "k8s.io/component-base/cli/flag" "k8s.io/klog/v2" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" @@ -53,6 +54,7 @@ import ( "github.com/projectsveltos/classifier/controllers" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/crd" "github.com/projectsveltos/libsveltos/lib/deployer" logs "github.com/projectsveltos/libsveltos/lib/logsettings" @@ -73,6 +75,8 @@ var ( managementClusterControlPlaneEndpoint string restConfigQPS float32 restConfigBurst int + managedClusterQPS float32 + managedClusterBurst int webhookPort int syncPeriod time.Duration version string @@ -113,9 +117,8 @@ func main() { ctrlOptions := getCtrlOptions(scheme) - restConfig := ctrl.GetConfigOrDie() - restConfig.QPS = restConfigQPS - restConfig.Burst = restConfigBurst + restConfig := getRestConfig() + clusterproxy.SetRestConfigLimits(managedClusterQPS, managedClusterBurst) mgr, err := ctrl.NewManager(restConfig, ctrlOptions) if err != nil { @@ -229,15 +232,25 @@ func initFlags(fs *pflag.FlagSet) { fs.BoolVar(&enableNatsWatcher, "enable-nats-watcher", false, "Enable NATS.io configuration watcher on sveltos-agents to listen for cloud events. Disabled by default.") - const defautlRestConfigQPS = 20 + const defautlRestConfigQPS = 40 fs.Float32Var(&restConfigQPS, "kube-api-qps", defautlRestConfigQPS, - fmt.Sprintf("Maximum queries per second from the controller client to the Kubernetes API server. Defaults to %d", - defautlRestConfigQPS)) + fmt.Sprintf("Maximum queries per second from the controller client to the management cluster Kubernetes API server. "+ + "Defaults to %d", defautlRestConfigQPS)) - const defaultRestConfigBurst = 30 + const defaultRestConfigBurst = 120 fs.IntVar(&restConfigBurst, "kube-api-burst", defaultRestConfigBurst, - fmt.Sprintf("Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default %d", - defaultRestConfigBurst)) + fmt.Sprintf("Maximum number of queries that should be allowed in one burst from the controller client to the "+ + "management cluster Kubernetes API server. Default %d", defaultRestConfigBurst)) + + const defaultManagedClusterRestConfigQPS = 20 + fs.Float32Var(&managedClusterQPS, "managed-cluster-kube-api-qps", defaultManagedClusterRestConfigQPS, + fmt.Sprintf("Maximum queries per second from the controller client to the Kubernetes API server of each managed cluster. "+ + "Defaults to %d", defaultManagedClusterRestConfigQPS)) + + const defaultManagedClusterRestConfigBurst = 60 + fs.IntVar(&managedClusterBurst, "managed-cluster-kube-api-burst", defaultManagedClusterRestConfigBurst, + fmt.Sprintf("Maximum number of queries that should be allowed in one burst from the controller client to the "+ + "Kubernetes API server of each managed cluster. Default %d", defaultManagedClusterRestConfigBurst)) const defaultWebhookPort = 9443 fs.IntVar(&webhookPort, "webhook-port", defaultWebhookPort, @@ -425,6 +438,14 @@ func getDiagnosticsOptions() metricsserver.Options { } } +// getRestConfig returns the rest.Config for the management cluster, with the configured QPS and burst. +func getRestConfig() *rest.Config { + restConfig := ctrl.GetConfigOrDie() + restConfig.QPS = restConfigQPS + restConfig.Burst = restConfigBurst + return restConfig +} + func getCtrlOptions(scheme *runtime.Scheme) ctrl.Options { return ctrl.Options{ Scheme: scheme,