From 74646a3218f1532398f867e7aa48401b89dd4fc5 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 4 Aug 2026 22:58:16 +0000 Subject: [PATCH 1/4] feat: configure kubelet through generated config --- docs/usages/configuration.md | 39 ++++++++++++++ go.mod | 12 ++--- go.sum | 24 ++++----- pkg/config/adapter.go | 19 +++++++ pkg/config/adapter_test.go | 43 +++++++++++++-- pkg/config/config.go | 28 ++++++++++ pkg/config/copy_test.go | 14 +++++ pkg/config/kubelet_config_test.go | 82 +++++++++++++++++++++++++++++ pkg/daemon/nodeoperator.go | 30 +++++++++-- pkg/daemon/nodeoperator_test.go | 87 +++++++++++++++++++++++++++++++ 10 files changed, 352 insertions(+), 26 deletions(-) create mode 100644 pkg/config/kubelet_config_test.go diff --git a/docs/usages/configuration.md b/docs/usages/configuration.md index 11d8084e..ccd992aa 100644 --- a/docs/usages/configuration.md +++ b/docs/usages/configuration.md @@ -142,6 +142,12 @@ At least one join or Azure authentication method must be configured. `azure.boot | `node.kubelet.clusterFQDN` | string | Kubernetes API server FQDN. Required for bootstrap token mode. | `example.hcp.canadacentral.azmk8s.io` | | `node.kubelet.caCertData` | string | Base64-encoded cluster CA data. Required for bootstrap token mode. | `` | | `node.kubelet.nodeIP` | string | Optional node IP override for kubelet `--node-ip`. | `10.0.0.4` | +| `node.kubelet.imageCredentialProvider.configPath` | string | Optional absolute path inside the nspawn machine to a kubelet exec image credential provider configuration file or supported configuration directory. Must be set with `binDir`. | `/etc/kubernetes/credential-provider.yaml` | +| `node.kubelet.imageCredentialProvider.binDir` | string | Optional absolute path inside the nspawn machine containing exec image credential provider binaries. Must be set with `configPath`. | `/usr/local/lib/kubelet-credential-providers` | + +Provider paths must be clean absolute machine paths without whitespace or systemd argument characters. Include the provider files in the OCI rootfs or expose them with read-only `bootstrap.additionalHostMounts`. + +The image credential provider executes a plugin to obtain short-lived pull credentials; it does not place registry passwords or tokens in the FlexNode configuration. Do not store static registry credentials in this file or provider configuration. ## Component Versions @@ -348,3 +354,36 @@ Use `bootstrap.additionalHostMounts` to expose host files or directories inside ``` Read-only entries render as systemd-nspawn `BindReadOnly=` directives; writable entries render as `Bind=` directives. + +### Image Pull Credential Provider + +Kubelet exec image credential providers can obtain short-lived registry credentials without storing tokens in the FlexNode config. The provider configuration and executable must exist inside the nspawn machine, either in the OCI rootfs or through host mounts: + +```json +{ + "bootstrap": { + "additionalHostMounts": [ + { + "source": "/opt/aks-flex-node/credential-provider/config.yaml", + "target": "/etc/kubernetes/credential-provider.yaml", + "readOnly": true + }, + { + "source": "/opt/aks-flex-node/credential-provider/bin", + "target": "/usr/local/lib/kubelet-credential-providers", + "readOnly": true + } + ] + }, + "node": { + "kubelet": { + "imageCredentialProvider": { + "configPath": "/etc/kubernetes/credential-provider.yaml", + "binDir": "/usr/local/lib/kubelet-credential-providers" + } + } + } +} +``` + +Provider binaries must be executable before the machine starts. Mount provider assets read-only unless the provider explicitly requires writable state. diff --git a/go.mod b/go.mod index f7009d23..8360aded 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,14 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.2 + github.com/Azure/unbounded v0.2.3-0.20260804222626-7dacc2cdd8ac github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.2 - k8s.io/api v0.36.2 - k8s.io/apimachinery v0.36.2 - k8s.io/client-go v0.36.2 + k8s.io/api v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/client-go v0.36.3 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 ) @@ -101,12 +101,12 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.36.0 // indirect + k8s.io/apiextensions-apiserver v0.36.3 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 // indirect oras.land/oras-go/v2 v2.6.2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index dd35efd4..3846ef3a 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.2 h1:uu5hYj20UBbrSAlf4qnDiMTGaR0xxsYnkdbTWTdBHX8= -github.com/Azure/unbounded v0.2.2/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw= +github.com/Azure/unbounded v0.2.3-0.20260804222626-7dacc2cdd8ac h1:duo0qxzi02CHxwaXo+btkls92ZBCDV6BfyG9UgqyO+A= +github.com/Azure/unbounded v0.2.3-0.20260804222626-7dacc2cdd8ac/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= @@ -348,14 +348,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= -k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= -k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= -k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= -k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= -k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= -k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= -k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= +k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0= +k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 h1:Sztf7ESG9tAXRW/ACJZjrj5jhdOUqS2KFRQT+CTvu78= @@ -370,7 +370,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index dfd302b3..5ec92755 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -42,6 +42,7 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { NodeIP: cfg.Node.Kubelet.NodeIP, Labels: cfg.Node.Labels, RegisterWithTaints: cfg.Node.Taints, + Configuration: kubeletConfiguration(cfg), }, CRI: agentconfig.CRIConfig{ Containerd: agentconfig.ContainerdConfig{ @@ -57,6 +58,13 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { }, } + if provider := cfg.Node.Kubelet.ImageCredentialProvider; provider != nil { + ac.Kubelet.ImageCredentialProvider = &agentconfig.ImageCredentialProvider{ + ConfigPath: provider.ConfigPath, + BinDir: provider.BinDir, + } + } + if cfg.Bootstrap.OfflineArtifacts.Source != "" { ac.OfflineArtifacts = &agentconfig.AgentOfflineArtifacts{ Source: cfg.Bootstrap.OfflineArtifacts.Source, @@ -103,6 +111,17 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { return ac } +func kubeletConfiguration(cfg *Config) map[string]any { + return map[string]any{ + "maxPods": cfg.Node.MaxPods, + "imageGCHighThresholdPercent": cfg.Node.Kubelet.ImageGCHighThreshold, + "imageGCLowThresholdPercent": cfg.Node.Kubelet.ImageGCLowThreshold, + "logging": map[string]any{ + "verbosity": cfg.Node.Kubelet.Verbosity, + }, + } +} + // ResolveMachineGoalState converts FlexNode config to the shared agent config // and resolves the nspawn machine goal state. Bootstrap and preflight both use // this helper so preflight validates the same sources that bootstrap consumes. diff --git a/pkg/config/adapter_test.go b/pkg/config/adapter_test.go index 510af5e4..82bdf6e5 100644 --- a/pkg/config/adapter_test.go +++ b/pkg/config/adapter_test.go @@ -19,12 +19,20 @@ func TestToAgentConfig_BootstrapToken(t *testing.T) { Components: ComponentsConfig{Kubernetes: "1.30.0"}, Networking: NetworkingConfig{DNSServiceIP: "10.0.0.10"}, Node: NodeConfig{ - Labels: map[string]string{"env": "test"}, - Taints: []string{"dedicated=infra:NoSchedule"}, + MaxPods: 42, + Labels: map[string]string{"env": "test"}, + Taints: []string{"dedicated=infra:NoSchedule"}, Kubelet: KubeletConfig{ - ClusterFQDN: "api.example.com:6443", - CACertData: "dGVzdC1jYS1kYXRh", - NodeIP: "10.225.0.4", + Verbosity: 4, + ImageGCHighThreshold: 90, + ImageGCLowThreshold: 75, + ClusterFQDN: "api.example.com:6443", + CACertData: "dGVzdC1jYS1kYXRh", + NodeIP: "10.225.0.4", + ImageCredentialProvider: &ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, }, }, } @@ -65,6 +73,31 @@ func TestToAgentConfig_BootstrapToken(t *testing.T) { if len(ac.Kubelet.RegisterWithTaints) != 1 || ac.Kubelet.RegisterWithTaints[0] != "dedicated=infra:NoSchedule" { t.Fatalf("Kubelet.RegisterWithTaints=%v, want [dedicated=infra:NoSchedule]", ac.Kubelet.RegisterWithTaints) } + if got := ac.Kubelet.Configuration["maxPods"]; got != 42 { + t.Fatalf("Kubelet.Configuration.maxPods=%v, want 42", got) + } + if got := ac.Kubelet.Configuration["imageGCHighThresholdPercent"]; got != 90 { + t.Fatalf("Kubelet.Configuration.imageGCHighThresholdPercent=%v, want 90", got) + } + if got := ac.Kubelet.Configuration["imageGCLowThresholdPercent"]; got != 75 { + t.Fatalf("Kubelet.Configuration.imageGCLowThresholdPercent=%v, want 75", got) + } + logging, ok := ac.Kubelet.Configuration["logging"].(map[string]any) + if !ok { + t.Fatalf("Kubelet.Configuration.logging=%T, want map[string]any", ac.Kubelet.Configuration["logging"]) + } + if logging["verbosity"] != 4 { + t.Fatalf("Kubelet.Configuration.logging=%v, want verbosity=4", logging) + } + if ac.Kubelet.ImageCredentialProvider == nil { + t.Fatal("Kubelet.ImageCredentialProvider=nil, want provider") + } + if ac.Kubelet.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" { + t.Fatalf("Kubelet.ImageCredentialProvider.ConfigPath=%q", ac.Kubelet.ImageCredentialProvider.ConfigPath) + } + if ac.Kubelet.ImageCredentialProvider.BinDir != "/usr/local/lib/kubelet-credential-providers" { + t.Fatalf("Kubelet.ImageCredentialProvider.BinDir=%q", ac.Kubelet.ImageCredentialProvider.BinDir) + } } func TestToAgentConfig_NodeName(t *testing.T) { diff --git a/pkg/config/config.go b/pkg/config/config.go index ecc670c5..f1a9025a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -251,6 +251,17 @@ type KubeletConfig struct { ClusterFQDN string `json:"clusterFQDN,omitempty"` // Kubernetes API server FQDN from AKS RP bootstrap data CACertData string `json:"caCertData"` // Base64-encoded CA certificate data NodeIP string `json:"nodeIP"` // IP address to advertise as the node's primary IP (--node-ip kubelet flag) + + // ImageCredentialProvider configures kubelet's exec image credential + // provider. The referenced paths are inside the nspawn machine. + ImageCredentialProvider *ImageCredentialProviderConfig `json:"imageCredentialProvider,omitempty"` +} + +// ImageCredentialProviderConfig identifies the exec image credential provider +// configuration and binary directory inside the nspawn machine. +type ImageCredentialProviderConfig struct { + ConfigPath string `json:"configPath"` + BinDir string `json:"binDir"` } // NetworkingConfig is the AKS RP networking contract used by the agent at runtime. @@ -882,6 +893,9 @@ func (c *Config) validate() error { if err := c.Bootstrap.validate(); err != nil { return err } + if err := c.Node.Kubelet.validate(); err != nil { + return err + } if err := c.validateAuthSettings(); err != nil { return err @@ -893,6 +907,20 @@ func (c *Config) validate() error { return nil } +func (c *KubeletConfig) validate() error { + kubelet := agentconfig.AgentKubeletConfig{} + if c.ImageCredentialProvider != nil { + kubelet.ImageCredentialProvider = &agentconfig.ImageCredentialProvider{ + ConfigPath: c.ImageCredentialProvider.ConfigPath, + BinDir: c.ImageCredentialProvider.BinDir, + } + } + if err := kubelet.Validate(); err != nil { + return fmt.Errorf("invalid node.kubelet configuration: %w", err) + } + return nil +} + func (c *Config) validateAuthSettings() error { armAuthMethodCount := 0 for _, m := range []bool{c.IsARCEnabled(), c.IsSPConfigured(), c.IsMIConfigured()} { diff --git a/pkg/config/copy_test.go b/pkg/config/copy_test.go index cae05009..114b0a98 100644 --- a/pkg/config/copy_test.go +++ b/pkg/config/copy_test.go @@ -17,6 +17,12 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) { Node: NodeConfig{ Labels: map[string]string{"l": "1"}, Taints: []string{"dedicated=infra:NoSchedule"}, + Kubelet: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, + }, }, } @@ -47,6 +53,9 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) { if cfg.Components.Gantry == nil || copy.Components.Gantry == nil || cfg.Components.Gantry == copy.Components.Gantry { t.Fatalf("Gantry pointer shared or nil") } + if cfg.Node.Kubelet.ImageCredentialProvider == nil || copy.Node.Kubelet.ImageCredentialProvider == nil || cfg.Node.Kubelet.ImageCredentialProvider == copy.Node.Kubelet.ImageCredentialProvider { + t.Fatalf("ImageCredentialProvider pointer shared or nil") + } // Maps should not be shared (validate via independent mutation behavior). cfg.Azure.Arc.Tags["k"] = "orig" @@ -67,6 +76,11 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) { t.Fatalf("Node.Labels shared; orig=%q, want %q", cfg.Node.Labels["l"], "orig") } + copy.Node.Kubelet.ImageCredentialProvider.ConfigPath = "/etc/kubernetes/other.yaml" + if cfg.Node.Kubelet.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" { + t.Fatal("ImageCredentialProvider shared between config copies") + } + // Taints slice should not be shared. if len(copy.Node.Taints) != 1 || copy.Node.Taints[0] != "dedicated=infra:NoSchedule" { t.Fatalf("Node.Taints copy=%v, want [dedicated=infra:NoSchedule]", copy.Node.Taints) diff --git a/pkg/config/kubelet_config_test.go b/pkg/config/kubelet_config_test.go new file mode 100644 index 00000000..c2692118 --- /dev/null +++ b/pkg/config/kubelet_config_test.go @@ -0,0 +1,82 @@ +package config + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestKubeletConfigJSON(t *testing.T) { + t.Parallel() + + data := []byte(`{ + "imageCredentialProvider": { + "configPath": "/etc/kubernetes/credential-provider.yaml", + "binDir": "/usr/local/lib/kubelet-credential-providers" + } + }`) + var config KubeletConfig + if err := json.Unmarshal(data, &config); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if config.ImageCredentialProvider == nil { + t.Fatal("ImageCredentialProvider=nil, want provider") + } + if config.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" || config.ImageCredentialProvider.BinDir != "/usr/local/lib/kubelet-credential-providers" { + t.Fatalf("ImageCredentialProvider=%#v", config.ImageCredentialProvider) + } +} + +func TestKubeletConfigValidate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + config KubeletConfig + wantErr string + }{ + {name: "empty"}, + { + name: "image credential provider", + config: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, + }, + }, + { + name: "missing provider config path", + config: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{BinDir: "/usr/bin"}, + }, + wantErr: "ConfigPath", + }, + { + name: "relative provider binary directory", + config: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "bin", + }, + }, + wantErr: "absolute path", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := tt.config.validate() + if tt.wantErr == "" { + if err != nil { + t.Fatalf("validate: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("validate error=%v, want substring %q", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/daemon/nodeoperator.go b/pkg/daemon/nodeoperator.go index f255cb9b..1a90a605 100644 --- a/pkg/daemon/nodeoperator.go +++ b/pkg/daemon/nodeoperator.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log/slog" + "maps" + "slices" "github.com/Azure/AKSFlexNode/pkg/aksmachine" "github.com/Azure/AKSFlexNode/pkg/config" @@ -84,9 +86,7 @@ func (o *nspawnNodeOperator) ApplyGoalState(ctx context.Context, log *slog.Logge // TODO: This per-goal config copy/mutation is not ideal. Refactor goal-state // resolution to avoid rewriting shared config-shaped data here. cfg := o.cfg.DeepCopy() - if goal.KubernetesVersion != "" { - cfg.Components.Kubernetes = goal.KubernetesVersion - } + applyMachineGoalToConfig(cfg, goal) oldMachine := active.Name newMachine := goalstates.AlternateMachine(oldMachine) log.Info("starting nspawn machine goal-state apply", @@ -114,6 +114,30 @@ func (o *nspawnNodeOperator) ApplyGoalState(ctx context.Context, log *slog.Logge return newState, nil } +func applyMachineGoalToConfig(cfg *config.Config, goal aksmachine.GoalState) { + if goal.KubernetesVersion != "" { + cfg.Components.Kubernetes = goal.KubernetesVersion + } + // Zero values represent fields omitted by the Machine API. Preserve local + // defaults in that case; non-nil empty labels or taints still explicitly + // clear those collections. + if goal.MaxPods != 0 { + cfg.Node.MaxPods = goal.MaxPods + } + if goal.NodeLabels != nil { + cfg.Node.Labels = maps.Clone(goal.NodeLabels) + } + if goal.NodeTaints != nil { + cfg.Node.Taints = slices.Clone(goal.NodeTaints) + } + if goal.KubeletConfig.ImageGCHighThreshold != 0 { + cfg.Node.Kubelet.ImageGCHighThreshold = goal.KubeletConfig.ImageGCHighThreshold + } + if goal.KubeletConfig.ImageGCLowThreshold != 0 { + cfg.Node.Kubelet.ImageGCLowThreshold = goal.KubeletConfig.ImageGCLowThreshold + } +} + func (o *nspawnNodeOperator) ResetNode(ctx context.Context, log *slog.Logger) error { return phases.ExecuteTask(ctx, log, ResetNode(log)) } diff --git a/pkg/daemon/nodeoperator_test.go b/pkg/daemon/nodeoperator_test.go index ae08a2cb..56201b2a 100644 --- a/pkg/daemon/nodeoperator_test.go +++ b/pkg/daemon/nodeoperator_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" + "github.com/Azure/AKSFlexNode/pkg/aksmachine" + "github.com/Azure/AKSFlexNode/pkg/config" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -57,6 +59,91 @@ func TestFindActiveMachine(t *testing.T) { } } +func TestApplyMachineGoalToConfig(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Components: config.ComponentsConfig{Kubernetes: "1.33.0"}, + Node: config.NodeConfig{ + MaxPods: 110, + Labels: map[string]string{"source": "local"}, + Taints: []string{"local=true:NoSchedule"}, + Kubelet: config.KubeletConfig{ + Verbosity: 4, + ImageGCHighThreshold: 85, + ImageGCLowThreshold: 80, + ImageCredentialProvider: &config.ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, + }, + }, + } + goal := aksmachine.GoalState{ + KubernetesVersion: "1.34.1", + MaxPods: 42, + NodeLabels: map[string]string{"source": "remote"}, + NodeTaints: []string{"remote=true:NoExecute"}, + KubeletConfig: aksmachine.KubeletConfig{ + ImageGCHighThreshold: 90, + ImageGCLowThreshold: 75, + }, + } + + applyMachineGoalToConfig(cfg, goal) + + if cfg.Components.Kubernetes != "1.34.1" || cfg.Node.MaxPods != 42 { + t.Fatalf("version=%q maxPods=%d, want 1.34.1 and 42", cfg.Components.Kubernetes, cfg.Node.MaxPods) + } + if cfg.Node.Labels["source"] != "remote" || cfg.Node.Taints[0] != "remote=true:NoExecute" { + t.Fatalf("labels=%v taints=%v, want remote settings", cfg.Node.Labels, cfg.Node.Taints) + } + if cfg.Node.Kubelet.ImageGCHighThreshold != 90 || cfg.Node.Kubelet.ImageGCLowThreshold != 75 { + t.Fatalf("kubelet GC thresholds=%d/%d, want 90/75", cfg.Node.Kubelet.ImageGCHighThreshold, cfg.Node.Kubelet.ImageGCLowThreshold) + } + if cfg.Node.Kubelet.Verbosity != 4 { + t.Fatal("local-only kubelet verbosity was not preserved") + } + if cfg.Node.Kubelet.ImageCredentialProvider == nil { + t.Fatal("local image credential provider was not preserved") + } + + goal.NodeLabels["source"] = "mutated" + goal.NodeTaints[0] = "mutated=true:NoSchedule" + if cfg.Node.Labels["source"] != "remote" || cfg.Node.Taints[0] != "remote=true:NoExecute" { + t.Fatal("applied config shares mutable goal state") + } +} + +func TestApplyMachineGoalToConfigPreservesSettingsOmittedByMachineAPI(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Components: config.ComponentsConfig{Kubernetes: "1.33.0"}, + Node: config.NodeConfig{ + MaxPods: 110, + Labels: map[string]string{"source": "local"}, + Taints: []string{"local=true:NoSchedule"}, + Kubelet: config.KubeletConfig{ + ImageGCHighThreshold: 85, + ImageGCLowThreshold: 80, + }, + }, + } + + applyMachineGoalToConfig(cfg, aksmachine.GoalState{KubernetesVersion: "1.34.1"}) + + if cfg.Components.Kubernetes != "1.34.1" { + t.Fatalf("Kubernetes version=%q, want 1.34.1", cfg.Components.Kubernetes) + } + if cfg.Node.MaxPods != 110 || cfg.Node.Labels["source"] != "local" || cfg.Node.Taints[0] != "local=true:NoSchedule" { + t.Fatalf("omitted machine settings replaced local values: %#v", cfg.Node) + } + if cfg.Node.Kubelet.ImageGCHighThreshold != 85 || cfg.Node.Kubelet.ImageGCLowThreshold != 80 { + t.Fatalf("omitted GC thresholds replaced local values: %#v", cfg.Node.Kubelet) + } +} + type testStateStore struct { state *State } From b990c58cf12c1c80d4a42d41300c615188c84989 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 4 Aug 2026 23:20:14 +0000 Subject: [PATCH 2/4] refactor: defer machine goal persistence --- pkg/daemon/nodeoperator.go | 30 ++---------- pkg/daemon/nodeoperator_test.go | 87 --------------------------------- 2 files changed, 3 insertions(+), 114 deletions(-) diff --git a/pkg/daemon/nodeoperator.go b/pkg/daemon/nodeoperator.go index 1a90a605..f255cb9b 100644 --- a/pkg/daemon/nodeoperator.go +++ b/pkg/daemon/nodeoperator.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "log/slog" - "maps" - "slices" "github.com/Azure/AKSFlexNode/pkg/aksmachine" "github.com/Azure/AKSFlexNode/pkg/config" @@ -86,7 +84,9 @@ func (o *nspawnNodeOperator) ApplyGoalState(ctx context.Context, log *slog.Logge // TODO: This per-goal config copy/mutation is not ideal. Refactor goal-state // resolution to avoid rewriting shared config-shaped data here. cfg := o.cfg.DeepCopy() - applyMachineGoalToConfig(cfg, goal) + if goal.KubernetesVersion != "" { + cfg.Components.Kubernetes = goal.KubernetesVersion + } oldMachine := active.Name newMachine := goalstates.AlternateMachine(oldMachine) log.Info("starting nspawn machine goal-state apply", @@ -114,30 +114,6 @@ func (o *nspawnNodeOperator) ApplyGoalState(ctx context.Context, log *slog.Logge return newState, nil } -func applyMachineGoalToConfig(cfg *config.Config, goal aksmachine.GoalState) { - if goal.KubernetesVersion != "" { - cfg.Components.Kubernetes = goal.KubernetesVersion - } - // Zero values represent fields omitted by the Machine API. Preserve local - // defaults in that case; non-nil empty labels or taints still explicitly - // clear those collections. - if goal.MaxPods != 0 { - cfg.Node.MaxPods = goal.MaxPods - } - if goal.NodeLabels != nil { - cfg.Node.Labels = maps.Clone(goal.NodeLabels) - } - if goal.NodeTaints != nil { - cfg.Node.Taints = slices.Clone(goal.NodeTaints) - } - if goal.KubeletConfig.ImageGCHighThreshold != 0 { - cfg.Node.Kubelet.ImageGCHighThreshold = goal.KubeletConfig.ImageGCHighThreshold - } - if goal.KubeletConfig.ImageGCLowThreshold != 0 { - cfg.Node.Kubelet.ImageGCLowThreshold = goal.KubeletConfig.ImageGCLowThreshold - } -} - func (o *nspawnNodeOperator) ResetNode(ctx context.Context, log *slog.Logger) error { return phases.ExecuteTask(ctx, log, ResetNode(log)) } diff --git a/pkg/daemon/nodeoperator_test.go b/pkg/daemon/nodeoperator_test.go index 56201b2a..ae08a2cb 100644 --- a/pkg/daemon/nodeoperator_test.go +++ b/pkg/daemon/nodeoperator_test.go @@ -4,8 +4,6 @@ import ( "context" "testing" - "github.com/Azure/AKSFlexNode/pkg/aksmachine" - "github.com/Azure/AKSFlexNode/pkg/config" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -59,91 +57,6 @@ func TestFindActiveMachine(t *testing.T) { } } -func TestApplyMachineGoalToConfig(t *testing.T) { - t.Parallel() - - cfg := &config.Config{ - Components: config.ComponentsConfig{Kubernetes: "1.33.0"}, - Node: config.NodeConfig{ - MaxPods: 110, - Labels: map[string]string{"source": "local"}, - Taints: []string{"local=true:NoSchedule"}, - Kubelet: config.KubeletConfig{ - Verbosity: 4, - ImageGCHighThreshold: 85, - ImageGCLowThreshold: 80, - ImageCredentialProvider: &config.ImageCredentialProviderConfig{ - ConfigPath: "/etc/kubernetes/credential-provider.yaml", - BinDir: "/usr/local/lib/kubelet-credential-providers", - }, - }, - }, - } - goal := aksmachine.GoalState{ - KubernetesVersion: "1.34.1", - MaxPods: 42, - NodeLabels: map[string]string{"source": "remote"}, - NodeTaints: []string{"remote=true:NoExecute"}, - KubeletConfig: aksmachine.KubeletConfig{ - ImageGCHighThreshold: 90, - ImageGCLowThreshold: 75, - }, - } - - applyMachineGoalToConfig(cfg, goal) - - if cfg.Components.Kubernetes != "1.34.1" || cfg.Node.MaxPods != 42 { - t.Fatalf("version=%q maxPods=%d, want 1.34.1 and 42", cfg.Components.Kubernetes, cfg.Node.MaxPods) - } - if cfg.Node.Labels["source"] != "remote" || cfg.Node.Taints[0] != "remote=true:NoExecute" { - t.Fatalf("labels=%v taints=%v, want remote settings", cfg.Node.Labels, cfg.Node.Taints) - } - if cfg.Node.Kubelet.ImageGCHighThreshold != 90 || cfg.Node.Kubelet.ImageGCLowThreshold != 75 { - t.Fatalf("kubelet GC thresholds=%d/%d, want 90/75", cfg.Node.Kubelet.ImageGCHighThreshold, cfg.Node.Kubelet.ImageGCLowThreshold) - } - if cfg.Node.Kubelet.Verbosity != 4 { - t.Fatal("local-only kubelet verbosity was not preserved") - } - if cfg.Node.Kubelet.ImageCredentialProvider == nil { - t.Fatal("local image credential provider was not preserved") - } - - goal.NodeLabels["source"] = "mutated" - goal.NodeTaints[0] = "mutated=true:NoSchedule" - if cfg.Node.Labels["source"] != "remote" || cfg.Node.Taints[0] != "remote=true:NoExecute" { - t.Fatal("applied config shares mutable goal state") - } -} - -func TestApplyMachineGoalToConfigPreservesSettingsOmittedByMachineAPI(t *testing.T) { - t.Parallel() - - cfg := &config.Config{ - Components: config.ComponentsConfig{Kubernetes: "1.33.0"}, - Node: config.NodeConfig{ - MaxPods: 110, - Labels: map[string]string{"source": "local"}, - Taints: []string{"local=true:NoSchedule"}, - Kubelet: config.KubeletConfig{ - ImageGCHighThreshold: 85, - ImageGCLowThreshold: 80, - }, - }, - } - - applyMachineGoalToConfig(cfg, aksmachine.GoalState{KubernetesVersion: "1.34.1"}) - - if cfg.Components.Kubernetes != "1.34.1" { - t.Fatalf("Kubernetes version=%q, want 1.34.1", cfg.Components.Kubernetes) - } - if cfg.Node.MaxPods != 110 || cfg.Node.Labels["source"] != "local" || cfg.Node.Taints[0] != "local=true:NoSchedule" { - t.Fatalf("omitted machine settings replaced local values: %#v", cfg.Node) - } - if cfg.Node.Kubelet.ImageGCHighThreshold != 85 || cfg.Node.Kubelet.ImageGCLowThreshold != 80 { - t.Fatalf("omitted GC thresholds replaced local values: %#v", cfg.Node.Kubelet) - } -} - type testStateStore struct { state *State } From aa802571b4d642c53d556f8402d5401b22213a25 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 21:20:51 +0000 Subject: [PATCH 3/4] chore: bump unbounded to v0.2.3-alpha.3 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8360aded..6f017b83 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260804222626-7dacc2cdd8ac + github.com/Azure/unbounded v0.2.3-alpha.3 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 3846ef3a..7e9dd9a4 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260804222626-7dacc2cdd8ac h1:duo0qxzi02CHxwaXo+btkls92ZBCDV6BfyG9UgqyO+A= -github.com/Azure/unbounded v0.2.3-0.20260804222626-7dacc2cdd8ac/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-alpha.3 h1:grxHFtQpulYHSFRusRTPZl//bPrHO9lifsJrjcFVrxo= +github.com/Azure/unbounded v0.2.3-alpha.3/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 15c895c50c29e3ea89fc0453a440c33ad5113c0a Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 21:29:25 +0000 Subject: [PATCH 4/4] fix: validate generated kubelet configuration --- pkg/config/config.go | 23 ++++++++- pkg/config/kubelet_config_test.go | 77 ++++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index f1a9025a..85e7b0ab 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -8,6 +8,7 @@ import ( "encoding/json" "encoding/pem" "fmt" + "math" "net/url" "os" "path/filepath" @@ -893,7 +894,7 @@ func (c *Config) validate() error { if err := c.Bootstrap.validate(); err != nil { return err } - if err := c.Node.Kubelet.validate(); err != nil { + if err := c.Node.validate(); err != nil { return err } @@ -907,7 +908,27 @@ func (c *Config) validate() error { return nil } +func (c *NodeConfig) validate() error { + if c.MaxPods < 0 || int64(c.MaxPods) > math.MaxInt32 { + return fmt.Errorf("node.maxPods must be between 0 and %d, inclusive", math.MaxInt32) + } + return c.Kubelet.validate() +} + func (c *KubeletConfig) validate() error { + if c.Verbosity < 0 || int64(c.Verbosity) > math.MaxInt32 { + return fmt.Errorf("node.kubelet.verbosity must be between 0 and %d, inclusive", math.MaxInt32) + } + if c.ImageGCHighThreshold < 0 || c.ImageGCHighThreshold > 100 { + return fmt.Errorf("node.kubelet.imageGCHighThreshold must be between 0 and 100, inclusive") + } + if c.ImageGCLowThreshold < 0 || c.ImageGCLowThreshold > 100 { + return fmt.Errorf("node.kubelet.imageGCLowThreshold must be between 0 and 100, inclusive") + } + if c.ImageGCLowThreshold >= c.ImageGCHighThreshold { + return fmt.Errorf("node.kubelet.imageGCLowThreshold must be less than node.kubelet.imageGCHighThreshold") + } + kubelet := agentconfig.AgentKubeletConfig{} if c.ImageCredentialProvider != nil { kubelet.ImageCredentialProvider = &agentconfig.ImageCredentialProvider{ diff --git a/pkg/config/kubelet_config_test.go b/pkg/config/kubelet_config_test.go index c2692118..99411502 100644 --- a/pkg/config/kubelet_config_test.go +++ b/pkg/config/kubelet_config_test.go @@ -36,6 +36,43 @@ func TestKubeletConfigValidate(t *testing.T) { wantErr string }{ {name: "empty"}, + { + name: "negative verbosity", + config: KubeletConfig{ + Verbosity: -1, + }, + wantErr: "verbosity must be between 0", + }, + { + name: "image GC high threshold above 100", + config: KubeletConfig{ + ImageGCHighThreshold: 101, + }, + wantErr: "imageGCHighThreshold must be between 0 and 100", + }, + { + name: "negative image GC low threshold", + config: KubeletConfig{ + ImageGCLowThreshold: -1, + }, + wantErr: "imageGCLowThreshold must be between 0 and 100", + }, + { + name: "image GC low threshold equals high threshold", + config: KubeletConfig{ + ImageGCHighThreshold: 90, + ImageGCLowThreshold: 90, + }, + wantErr: "imageGCLowThreshold must be less than node.kubelet.imageGCHighThreshold", + }, + { + name: "image GC low threshold above high threshold", + config: KubeletConfig{ + ImageGCHighThreshold: 80, + ImageGCLowThreshold: 90, + }, + wantErr: "imageGCLowThreshold must be less than node.kubelet.imageGCHighThreshold", + }, { name: "image credential provider", config: KubeletConfig{ @@ -67,7 +104,9 @@ func TestKubeletConfigValidate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - err := tt.config.validate() + cfg := Config{Node: NodeConfig{Kubelet: tt.config}} + cfg.setNodeDefaults() + err := cfg.Node.Kubelet.validate() if tt.wantErr == "" { if err != nil { t.Fatalf("validate: %v", err) @@ -80,3 +119,39 @@ func TestKubeletConfigValidate(t *testing.T) { }) } } + +func TestNodeConfigValidateMaxPods(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + maxPods int + wantErr bool + }{ + {name: "zero"}, + {name: "positive", maxPods: 110}, + {name: "negative", maxPods: -1, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cfg := NodeConfig{ + MaxPods: tt.maxPods, + Kubelet: KubeletConfig{ + Verbosity: 2, + ImageGCHighThreshold: 85, + ImageGCLowThreshold: 80, + }, + } + err := cfg.validate() + if tt.wantErr && err == nil { + t.Fatal("validate succeeded, want error") + } + if !tt.wantErr && err != nil { + t.Fatalf("validate: %v", err) + } + }) + } +}