From e54eeac429eeaa0af8dcc5b69634c9bd6842eaca Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 29 Jul 2026 04:52:04 +0000 Subject: [PATCH 01/16] feat: support AKS LocalDNS profiles --- docs/usages/configuration.md | 44 +++++++ hack/e2e/lib/node-join-msi.sh | 32 ++++- hack/e2e/lib/validate.sh | 26 ++++ pkg/config/adapter.go | 19 ++- pkg/config/config.go | 8 +- pkg/config/localdns.go | 191 ++++++++++++++++++++++++++++ pkg/config/localdns_adapter_test.go | 50 ++++++++ pkg/config/localdns_test.go | 64 ++++++++++ 8 files changed, 430 insertions(+), 4 deletions(-) create mode 100644 pkg/config/localdns.go create mode 100644 pkg/config/localdns_adapter_test.go create mode 100644 pkg/config/localdns_test.go diff --git a/docs/usages/configuration.md b/docs/usages/configuration.md index 11d8084e..87bce17a 100644 --- a/docs/usages/configuration.md +++ b/docs/usages/configuration.md @@ -122,6 +122,50 @@ At least one join or Azure authentication method must be configured. `azure.boot |------|------|-------------|--------------| | `networking.dnsServiceIP` | string | Cluster DNS service IP. | `10.0.0.10` | | `networking.cniVersion` | string | Optional CNI plugin version override. | `v1.6.2` | +| `networking.localDNS` | object | Optional AKS LocalDNS profile using the same `mode`, `vnetDNSOverrides`, and `kubeDNSOverrides` shape accepted by `az aks nodepool --localdns-config`. | `{ "mode": "Required" }` | + +### AKS LocalDNS + +AKS Flex Node accepts the official AKS LocalDNS JSON profile under +`networking.localDNS`. See [Configure LocalDNS in +AKS](https://learn.microsoft.com/azure/aks/localdns-custom) for the supported +fields and values. `Required` enables LocalDNS, `Disabled` disables it through +repave, and `Preferred` validates the profile without enabling the service. + +```json +{ + "networking": { + "dnsServiceIP": "10.0.0.10", + "localDNS": { + "mode": "Required", + "vnetDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "ForceTCP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } +} +``` ## Node diff --git a/hack/e2e/lib/node-join-msi.sh b/hack/e2e/lib/node-join-msi.sh index e9167574..053d3593 100644 --- a/hack/e2e/lib/node-join-msi.sh +++ b/hack/e2e/lib/node-join-msi.sh @@ -55,7 +55,8 @@ node_join_msi() { "node": { "kubelet": { "clusterFQDN": "${server_url}", - "caCertData": "${ca_cert_data}" + "caCertData": "${ca_cert_data}", + "nodeIP": "${vm_ip}" } }, "agent": { @@ -67,6 +68,35 @@ node_join_msi() { }, "requireMachineRegistration": true }, + "networking": { + "localDNS": { + "mode": "Required", + "vnetDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "ForceTCP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + }, "components": { "kubernetes": "${E2E_KUBERNETES_VERSION}", "containerd": "${E2E_CONTAINERD_VERSION}", diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 02ab9f6c..44ddd7a3 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -168,6 +168,31 @@ REMOTE } # --------------------------------------------------------------------------- +# validate_localdns_status - Verify nspawn LocalDNS on the selected VM. +validate_localdns_status() { + local vm_ip="$1" + log_info "Validating LocalDNS on ${vm_ip}..." + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail +machine=$(sudo machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}') +test -n "${machine}" +sudo systemd-run --quiet --pipe --wait --machine="${machine}" systemctl is-active --quiet localdns.service +sudo systemd-run --quiet --pipe --wait --machine="${machine}" grep -qx 'nameserver 169.254.10.10' /etc/resolv.conf +sudo ip address show dev localdns | grep -q '169.254.10.10/32' +sudo ip address show dev localdns | grep -q '169.254.10.11/32' +for chain in OUTPUT PREROUTING; do + for address in 169.254.10.10 169.254.10.11; do + for protocol in tcp udp; do + sudo iptables -w -t raw -C "${chain}" -m comment \ + --comment 'unbounded-localdns: skip conntrack' \ + -p "${protocol}" -d "${address}" --dport 53 -j NOTRACK + done + done +done +REMOTE + log_success "LocalDNS validation passed on ${vm_ip}" +} + # validate_all_nodes - Check all MSI, token, offline, and kubeadm VMs joined # --------------------------------------------------------------------------- validate_all_nodes() { @@ -206,6 +231,7 @@ validate_all_nodes() { validate_node_ip "${token_vm_name}" "${token_vm_private_ip}" || failed=1 validate_node_ip "${offline_vm_name}" "${offline_vm_private_ip}" || failed=1 validate_npd_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 + validate_localdns_status "${msi_vm_ip}" || failed=1 validate_npd_status "${token_vm_name}" "${token_vm_ip}" || failed=1 # TODO: re-enable once NPD is included in the upstream Unbounded bootstrap # artifact bundle and resolver used by offline artifact mode. diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index dfd302b3..f9e16a30 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "maps" agentconfig "github.com/Azure/unbounded/pkg/agent/config" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -26,6 +27,11 @@ const ( // // cfg.Node.Kubelet.ClusterFQDN and cfg.Node.Kubelet.CACertData must be populated. func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { + labels := maps.Clone(cfg.Node.Labels) + if labels == nil { + labels = map[string]string{} + } + ac := &agentconfig.AgentConfig{ MachineName: machineName, NodeName: cfg.Agent.NodeName, @@ -40,7 +46,7 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { Kubelet: agentconfig.AgentKubeletConfig{ ApiServer: cfg.APIServerURL(), NodeIP: cfg.Node.Kubelet.NodeIP, - Labels: cfg.Node.Labels, + Labels: labels, RegisterWithTaints: cfg.Node.Taints, }, CRI: agentconfig.CRIConfig{ @@ -57,6 +63,17 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { }, } + if profile := cfg.Networking.LocalDNS; profile != nil { + corefile, _ := profile.CorefileTemplate() // Config validation runs before adaptation. + ac.LocalDNS = &agentconfig.AgentLocalDNSConfig{ + Enabled: profile.Enabled(), + CorefileTemplate: corefile, + } + if profile.Enabled() { + labels["kubernetes.azure.com/localdns"] = "enabled" + } + } + if cfg.Bootstrap.OfflineArtifacts.Source != "" { ac.OfflineArtifacts = &agentconfig.AgentOfflineArtifacts{ Source: cfg.Bootstrap.OfflineArtifacts.Source, diff --git a/pkg/config/config.go b/pkg/config/config.go index ecc670c5..98bdb546 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -255,8 +255,9 @@ type KubeletConfig struct { // NetworkingConfig is the AKS RP networking contract used by the agent at runtime. type NetworkingConfig struct { - DNSServiceIP string `json:"dnsServiceIP,omitempty"` // Cluster DNS service IP (default: 10.0.0.10 for AKS) - CNIVersion string `json:"cniVersion,omitempty"` + DNSServiceIP string `json:"dnsServiceIP,omitempty"` // Cluster DNS service IP (default: 10.0.0.10 for AKS) + CNIVersion string `json:"cniVersion,omitempty"` + LocalDNS *LocalDNSProfile `json:"localDNS,omitempty"` } // NPDConfig holds configuration settings for the Node Problem Detector (NPD). @@ -882,6 +883,9 @@ func (c *Config) validate() error { if err := c.Bootstrap.validate(); err != nil { return err } + if err := c.Networking.LocalDNS.validate(); err != nil { + return fmt.Errorf("invalid networking.localDNS: %w", err) + } if err := c.validateAuthSettings(); err != nil { return err diff --git a/pkg/config/localdns.go b/pkg/config/localdns.go new file mode 100644 index 00000000..d552c806 --- /dev/null +++ b/pkg/config/localdns.go @@ -0,0 +1,191 @@ +package config + +import ( + "errors" + "fmt" + "sort" + "strings" +) + +const ( + LocalDNSModeRequired = "Required" + LocalDNSModePreferred = "Preferred" + LocalDNSModeDisabled = "Disabled" +) + +// LocalDNSProfile mirrors the AKS node-pool LocalDNS configuration contract. +type LocalDNSProfile struct { + Mode string `json:"mode"` + VnetDNSOverrides map[string]LocalDNSOverride `json:"vnetDNSOverrides,omitempty"` + KubeDNSOverrides map[string]LocalDNSOverride `json:"kubeDNSOverrides,omitempty"` +} + +// LocalDNSOverride configures one CoreDNS server block. +type LocalDNSOverride struct { + QueryLogging string `json:"queryLogging,omitempty"` + Protocol string `json:"protocol,omitempty"` + ForwardDestination string `json:"forwardDestination,omitempty"` + ForwardPolicy string `json:"forwardPolicy,omitempty"` + MaxConcurrent int `json:"maxConcurrent,omitempty"` + CacheDurationInSeconds int `json:"cacheDurationInSeconds,omitempty"` + ServeStaleDurationInSeconds int `json:"serveStaleDurationInSeconds,omitempty"` + ServeStale string `json:"serveStale,omitempty"` +} + +func (p *LocalDNSProfile) validate() error { + if p == nil { + return nil + } + if p.Mode != LocalDNSModeRequired && p.Mode != LocalDNSModePreferred && p.Mode != LocalDNSModeDisabled { + return fmt.Errorf("mode must be Required, Preferred, or Disabled") + } + var errs []error + for class, overrides := range map[string]map[string]LocalDNSOverride{ + "vnetDNSOverrides": p.VnetDNSOverrides, + "kubeDNSOverrides": p.KubeDNSOverrides, + } { + for zone, override := range overrides { + if strings.TrimSpace(zone) == "" || strings.ContainsAny(zone, "{} \t\r\n") { + errs = append(errs, fmt.Errorf("%s zone %q is invalid", class, zone)) + } + if err := override.validate(); err != nil { + errs = append(errs, fmt.Errorf("%s[%q]: %w", class, zone, err)) + } + } + } + return errors.Join(errs...) +} + +func (o LocalDNSOverride) validate() error { + var errs []error + if o.QueryLogging != "" && o.QueryLogging != "Error" && o.QueryLogging != "Log" { + errs = append(errs, fmt.Errorf("queryLogging must be Error or Log")) + } + if o.Protocol != "" && o.Protocol != "PreferUDP" && o.Protocol != "ForceTCP" { + errs = append(errs, fmt.Errorf("protocol must be PreferUDP or ForceTCP")) + } + if o.ForwardDestination != "" && o.ForwardDestination != "VnetDNS" && o.ForwardDestination != "ClusterCoreDNS" { + errs = append(errs, fmt.Errorf("forwardDestination must be VnetDNS or ClusterCoreDNS")) + } + if o.ForwardPolicy != "" && o.ForwardPolicy != "Sequential" && o.ForwardPolicy != "RoundRobin" && o.ForwardPolicy != "Random" { + errs = append(errs, fmt.Errorf("forwardPolicy must be Sequential, RoundRobin, or Random")) + } + if o.ServeStale != "" && o.ServeStale != "Disable" && o.ServeStale != "Verify" && o.ServeStale != "Immediate" { + errs = append(errs, fmt.Errorf("serveStale must be Disable, Verify, or Immediate")) + } + for name, value := range map[string]int{ + "maxConcurrent": o.MaxConcurrent, + "cacheDurationInSeconds": o.CacheDurationInSeconds, + "serveStaleDurationInSeconds": o.ServeStaleDurationInSeconds, + } { + if value < 0 { + errs = append(errs, fmt.Errorf("%s must not be negative", name)) + } + } + return errors.Join(errs...) +} + +// Enabled reports whether the profile requires LocalDNS installation. +func (p *LocalDNSProfile) Enabled() bool { + return p != nil && p.Mode == LocalDNSModeRequired +} + +// CorefileTemplate renders AKS LocalDNS policy into an Unbounded Corefile template. +func (p *LocalDNSProfile) CorefileTemplate() (string, error) { + if p == nil { + return "", nil + } + if err := p.validate(); err != nil { + return "", err + } + if !p.Enabled() { + return "", nil + } + + vnet := p.VnetDNSOverrides + if len(vnet) == 0 { + vnet = map[string]LocalDNSOverride{".": {ForwardDestination: "VnetDNS"}} + } + kube := p.KubeDNSOverrides + if len(kube) == 0 { + kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} + } + + var out strings.Builder + out.WriteString("health-check.localdns.local:53 {\n bind {{ .NodeListenerIP }} {{ .ClusterListenerIP }}\n whoami\n}\n\n") + renderLocalDNSBlocks(&out, vnet, "{{ .NodeListenerIP }}", "VnetDNS") + renderLocalDNSBlocks(&out, kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS") + return out.String(), nil +} + +func renderLocalDNSBlocks(out *strings.Builder, overrides map[string]LocalDNSOverride, listener, defaultDestination string) { + zones := make([]string, 0, len(overrides)) + for zone := range overrides { + zones = append(zones, zone) + } + sort.Strings(zones) + for _, zone := range zones { + o := withLocalDNSDefaults(overrides[zone], defaultDestination) + fmt.Fprintf(out, "%s:53 {\n", zone) + if o.QueryLogging == "Log" { + out.WriteString(" log\n") + } else { + out.WriteString(" errors\n") + } + fmt.Fprintf(out, " bind %s\n", listener) + upstream := "{{ .NodeUpstreamIPsJoined }}" + if o.ForwardDestination == "ClusterCoreDNS" { + upstream = "{{ .ClusterDNSServiceIP }}" + } + fmt.Fprintf(out, " forward . %s {\n", upstream) + if o.Protocol == "ForceTCP" { + out.WriteString(" force_tcp\n") + } + fmt.Fprintf(out, " policy %s\n max_concurrent %d\n }\n", localDNSForwardPolicy(o.ForwardPolicy), o.MaxConcurrent) + fmt.Fprintf(out, " ready %s:8181\n", listener) + fmt.Fprintf(out, " cache %d {\n success 9984\n denial 9984\n", o.CacheDurationInSeconds) + if o.ServeStale != "Disable" { + fmt.Fprintf(out, " serve_stale %ds %s\n", o.ServeStaleDurationInSeconds, strings.ToLower(o.ServeStale)) + } + out.WriteString(" servfail 0\n }\n loop\n prometheus {{ .MetricsAddress }}\n}\n\n") + } +} + +func withLocalDNSDefaults(o LocalDNSOverride, destination string) LocalDNSOverride { + if o.QueryLogging == "" { + o.QueryLogging = "Error" + } + if o.Protocol == "" { + o.Protocol = "ForceTCP" + } + if o.ForwardDestination == "" { + o.ForwardDestination = destination + } + if o.ForwardPolicy == "" { + o.ForwardPolicy = "Sequential" + } + if o.MaxConcurrent == 0 { + o.MaxConcurrent = 1000 + } + if o.CacheDurationInSeconds == 0 { + o.CacheDurationInSeconds = 3600 + } + if o.ServeStaleDurationInSeconds == 0 { + o.ServeStaleDurationInSeconds = 3600 + } + if o.ServeStale == "" { + o.ServeStale = "Immediate" + } + return o +} + +func localDNSForwardPolicy(policy string) string { + switch policy { + case "RoundRobin": + return "round_robin" + case "Random": + return "random" + default: + return "sequential" + } +} diff --git a/pkg/config/localdns_adapter_test.go b/pkg/config/localdns_adapter_test.go new file mode 100644 index 00000000..eba36356 --- /dev/null +++ b/pkg/config/localdns_adapter_test.go @@ -0,0 +1,50 @@ +package config + +import ( + "strings" + "testing" +) + +func TestToAgentConfigLocalDNS(t *testing.T) { + t.Parallel() + + cfg := &Config{ + Networking: NetworkingConfig{ + DNSServiceIP: "10.0.0.10", + LocalDNS: &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + KubeDNSOverrides: map[string]LocalDNSOverride{ + ".": {ForwardDestination: "ClusterCoreDNS"}, + }, + }, + }, + Node: NodeConfig{Labels: map[string]string{"example": "value"}}, + } + + got := ToAgentConfig(cfg, "kube1") + if got.LocalDNS == nil || !got.LocalDNS.Enabled { + t.Fatalf("LocalDNS = %#v, want enabled", got.LocalDNS) + } + if !strings.Contains(got.LocalDNS.CorefileTemplate, "{{ .ClusterDNSServiceIP }}") { + t.Fatalf("CorefileTemplate missing cluster DNS placeholder:\n%s", got.LocalDNS.CorefileTemplate) + } + if got.Kubelet.Labels["kubernetes.azure.com/localdns"] != "enabled" { + t.Fatalf("LocalDNS node label missing: %#v", got.Kubelet.Labels) + } + if got.Cluster.ClusterDNS != "10.0.0.10" { + t.Fatalf("ClusterDNS = %q, want original service IP", got.Cluster.ClusterDNS) + } +} + +func TestToAgentConfigDisabledLocalDNS(t *testing.T) { + t.Parallel() + + cfg := &Config{Networking: NetworkingConfig{ + DNSServiceIP: "10.0.0.10", + LocalDNS: &LocalDNSProfile{Mode: LocalDNSModeDisabled}, + }} + got := ToAgentConfig(cfg, "kube1") + if got.LocalDNS == nil || got.LocalDNS.Enabled { + t.Fatalf("LocalDNS = %#v, want explicitly disabled", got.LocalDNS) + } +} diff --git a/pkg/config/localdns_test.go b/pkg/config/localdns_test.go new file mode 100644 index 00000000..04e92abe --- /dev/null +++ b/pkg/config/localdns_test.go @@ -0,0 +1,64 @@ +package config + +import ( + "strings" + "testing" +) + +func TestLocalDNSProfileCorefileTemplate(t *testing.T) { + t.Parallel() + + profile := &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + VnetDNSOverrides: map[string]LocalDNSOverride{ + ".": {ForwardDestination: "VnetDNS", Protocol: "PreferUDP"}, + }, + KubeDNSOverrides: map[string]LocalDNSOverride{ + ".": {ForwardDestination: "ClusterCoreDNS", Protocol: "ForceTCP", ForwardPolicy: "RoundRobin"}, + }, + } + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatalf("CorefileTemplate() error = %v", err) + } + for _, want := range []string{ + "bind {{ .NodeListenerIP }}", + "forward . {{ .NodeUpstreamIPsJoined }}", + "bind {{ .ClusterListenerIP }}", + "forward . {{ .ClusterDNSServiceIP }}", + "force_tcp", + "policy round_robin", + } { + if !strings.Contains(got, want) { + t.Fatalf("CorefileTemplate() missing %q:\n%s", want, got) + } + } +} + +func TestLocalDNSProfileValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + profile *LocalDNSProfile + wantErr string + }{ + {name: "required", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired}}, + {name: "preferred", profile: &LocalDNSProfile{Mode: LocalDNSModePreferred}}, + {name: "disabled", profile: &LocalDNSProfile{Mode: LocalDNSModeDisabled}}, + {name: "invalid mode", profile: &LocalDNSProfile{Mode: "On"}, wantErr: "mode"}, + {name: "invalid protocol", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {Protocol: "TLS"}}}, wantErr: "protocol"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + err := test.profile.validate() + if test.wantErr == "" && err != nil { + t.Fatalf("validate() error = %v", err) + } + if test.wantErr != "" && (err == nil || !strings.Contains(err.Error(), test.wantErr)) { + t.Fatalf("validate() error = %v, want %q", err, test.wantErr) + } + }) + } +} From f918eefddc9476b6b8dba0cf25c6d526298c590e Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 29 Jul 2026 05:08:51 +0000 Subject: [PATCH 02/16] refactor: render LocalDNS blocks with templates --- pkg/config/localdns.go | 105 +++++++++++++++++++++++++++++++---------- 1 file changed, 81 insertions(+), 24 deletions(-) diff --git a/pkg/config/localdns.go b/pkg/config/localdns.go index d552c806..4f487d5c 100644 --- a/pkg/config/localdns.go +++ b/pkg/config/localdns.go @@ -1,10 +1,12 @@ package config import ( + "bytes" "errors" "fmt" "sort" "strings" + "text/template" ) const ( @@ -90,6 +92,53 @@ func (p *LocalDNSProfile) Enabled() bool { return p != nil && p.Mode == LocalDNSModeRequired } +const aksLocalDNSCorefileTemplate = `health-check.localdns.local:53 { + bind {{.NodeListener}} {{.ClusterListener}} + whoami +} + +{{range .Blocks}}{{.Zone}}:53 { + {{if .LogQueries}}log{{else}}errors{{end}} + bind {{.Listener}} + forward . {{.Upstream}} { + {{if .ForceTCP}}force_tcp + {{end}}policy {{.ForwardPolicy}} + max_concurrent {{.MaxConcurrent}} + } + ready {{.Listener}}:8181 + cache {{.CacheDuration}} { + success 9984 + denial 9984 + {{if .ServeStale}}serve_stale {{.ServeStaleDuration}}s {{.ServeStalePolicy}} + {{end}}servfail 0 + } + loop + prometheus {{$.MetricsAddress}} +} + +{{end}}` + +type localDNSCorefileData struct { + NodeListener string + ClusterListener string + MetricsAddress string + Blocks []localDNSCorefileBlock +} + +type localDNSCorefileBlock struct { + Zone string + Listener string + Upstream string + LogQueries bool + ForceTCP bool + ForwardPolicy string + MaxConcurrent int + CacheDuration int + ServeStale bool + ServeStaleDuration int + ServeStalePolicy string +} + // CorefileTemplate renders AKS LocalDNS policy into an Unbounded Corefile template. func (p *LocalDNSProfile) CorefileTemplate() (string, error) { if p == nil { @@ -111,44 +160,52 @@ func (p *LocalDNSProfile) CorefileTemplate() (string, error) { kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} } - var out strings.Builder - out.WriteString("health-check.localdns.local:53 {\n bind {{ .NodeListenerIP }} {{ .ClusterListenerIP }}\n whoami\n}\n\n") - renderLocalDNSBlocks(&out, vnet, "{{ .NodeListenerIP }}", "VnetDNS") - renderLocalDNSBlocks(&out, kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS") + data := localDNSCorefileData{ + NodeListener: "{{ .NodeListenerIP }}", + ClusterListener: "{{ .ClusterListenerIP }}", + MetricsAddress: "{{ .MetricsAddress }}", + Blocks: append(localDNSCorefileBlocks(vnet, "{{ .NodeListenerIP }}", "VnetDNS"), localDNSCorefileBlocks(kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS")...), + } + tmpl, err := template.New("aks-localdns-corefile").Parse(aksLocalDNSCorefileTemplate) + if err != nil { + return "", fmt.Errorf("parse AKS LocalDNS Corefile template: %w", err) + } + var out bytes.Buffer + if err := tmpl.Execute(&out, data); err != nil { + return "", fmt.Errorf("render AKS LocalDNS Corefile template: %w", err) + } return out.String(), nil } -func renderLocalDNSBlocks(out *strings.Builder, overrides map[string]LocalDNSOverride, listener, defaultDestination string) { +func localDNSCorefileBlocks(overrides map[string]LocalDNSOverride, listener, defaultDestination string) []localDNSCorefileBlock { zones := make([]string, 0, len(overrides)) for zone := range overrides { zones = append(zones, zone) } sort.Strings(zones) + + blocks := make([]localDNSCorefileBlock, 0, len(zones)) for _, zone := range zones { o := withLocalDNSDefaults(overrides[zone], defaultDestination) - fmt.Fprintf(out, "%s:53 {\n", zone) - if o.QueryLogging == "Log" { - out.WriteString(" log\n") - } else { - out.WriteString(" errors\n") - } - fmt.Fprintf(out, " bind %s\n", listener) upstream := "{{ .NodeUpstreamIPsJoined }}" if o.ForwardDestination == "ClusterCoreDNS" { upstream = "{{ .ClusterDNSServiceIP }}" } - fmt.Fprintf(out, " forward . %s {\n", upstream) - if o.Protocol == "ForceTCP" { - out.WriteString(" force_tcp\n") - } - fmt.Fprintf(out, " policy %s\n max_concurrent %d\n }\n", localDNSForwardPolicy(o.ForwardPolicy), o.MaxConcurrent) - fmt.Fprintf(out, " ready %s:8181\n", listener) - fmt.Fprintf(out, " cache %d {\n success 9984\n denial 9984\n", o.CacheDurationInSeconds) - if o.ServeStale != "Disable" { - fmt.Fprintf(out, " serve_stale %ds %s\n", o.ServeStaleDurationInSeconds, strings.ToLower(o.ServeStale)) - } - out.WriteString(" servfail 0\n }\n loop\n prometheus {{ .MetricsAddress }}\n}\n\n") - } + blocks = append(blocks, localDNSCorefileBlock{ + Zone: zone, + Listener: listener, + Upstream: upstream, + LogQueries: o.QueryLogging == "Log", + ForceTCP: o.Protocol == "ForceTCP", + ForwardPolicy: localDNSForwardPolicy(o.ForwardPolicy), + MaxConcurrent: o.MaxConcurrent, + CacheDuration: o.CacheDurationInSeconds, + ServeStale: o.ServeStale != "Disable", + ServeStaleDuration: o.ServeStaleDurationInSeconds, + ServeStalePolicy: strings.ToLower(o.ServeStale), + }) + } + return blocks } func withLocalDNSDefaults(o LocalDNSOverride, destination string) LocalDNSOverride { From e7d2aea8c301d1e33b89b3b24681791380f15ecf Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 29 Jul 2026 05:12:04 +0000 Subject: [PATCH 03/16] test: port AgentBaker LocalDNS policy cases --- pkg/config/localdns_test.go | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/pkg/config/localdns_test.go b/pkg/config/localdns_test.go index 04e92abe..7687ec89 100644 --- a/pkg/config/localdns_test.go +++ b/pkg/config/localdns_test.go @@ -35,6 +35,80 @@ func TestLocalDNSProfileCorefileTemplate(t *testing.T) { } } +func TestLocalDNSProfileCorefileOptions(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + override LocalDNSOverride + want []string + wantAbsent []string + }{ + { + name: "VnetDNS PreferUDP sequential immediate", + override: LocalDNSOverride{QueryLogging: "Log", Protocol: "PreferUDP", ForwardDestination: "VnetDNS", ForwardPolicy: "Sequential", MaxConcurrent: 1000, CacheDurationInSeconds: 30, ServeStaleDurationInSeconds: 60, ServeStale: "Immediate"}, + want: []string{"log", "forward . {{ .NodeUpstreamIPsJoined }}", "policy sequential", "max_concurrent 1000", "cache 30", "serve_stale 60s immediate"}, + wantAbsent: []string{"force_tcp"}, + }, + { + name: "ClusterCoreDNS ForceTCP round robin verify", + override: LocalDNSOverride{QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", ForwardPolicy: "RoundRobin", ServeStale: "Verify"}, + want: []string{"errors", "forward . {{ .ClusterDNSServiceIP }}", "force_tcp", "policy round_robin", "serve_stale 3600s verify"}, + }, + { + name: "random without stale", + override: LocalDNSOverride{ForwardDestination: "ClusterCoreDNS", ForwardPolicy: "Random", ServeStale: "Disable"}, + want: []string{"policy random"}, + wantAbsent: []string{"serve_stale"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + profile := &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + VnetDNSOverrides: map[string]LocalDNSOverride{".": test.override}, + KubeDNSOverrides: map[string]LocalDNSOverride{".": test.override}, + } + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatalf("CorefileTemplate() error = %v", err) + } + for _, want := range test.want { + if !strings.Contains(got, want) { + t.Errorf("CorefileTemplate() missing %q:\n%s", want, got) + } + } + for _, unwanted := range test.wantAbsent { + if strings.Contains(got, unwanted) { + t.Errorf("CorefileTemplate() unexpectedly contains %q:\n%s", unwanted, got) + } + } + }) + } +} + +func TestLocalDNSProfileCorefileZoneOrder(t *testing.T) { + t.Parallel() + + profile := &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{ + "z.example": {}, + ".": {}, + "a.example": {}, + }} + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatal(err) + } + root := strings.Index(got, ".:53 {") + a := strings.Index(got, "a.example:53 {") + z := strings.Index(got, "z.example:53 {") + if root < 0 || a < root || z < a { + t.Fatalf("zones are not deterministic: root=%d a=%d z=%d\n%s", root, a, z, got) + } +} + func TestLocalDNSProfileValidation(t *testing.T) { t.Parallel() @@ -48,6 +122,11 @@ func TestLocalDNSProfileValidation(t *testing.T) { {name: "disabled", profile: &LocalDNSProfile{Mode: LocalDNSModeDisabled}}, {name: "invalid mode", profile: &LocalDNSProfile{Mode: "On"}, wantErr: "mode"}, {name: "invalid protocol", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {Protocol: "TLS"}}}, wantErr: "protocol"}, + {name: "invalid logging", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {QueryLogging: "Debug"}}}, wantErr: "queryLogging"}, + {name: "invalid destination", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ForwardDestination: "External"}}}, wantErr: "forwardDestination"}, + {name: "invalid policy", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ForwardPolicy: "First"}}}, wantErr: "forwardPolicy"}, + {name: "invalid stale", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ServeStale: "Always"}}}, wantErr: "serveStale"}, + {name: "negative cache", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {CacheDurationInSeconds: -1}}}, wantErr: "cacheDurationInSeconds"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { From c9bc8f94a043248b2ece7f965561706edd1d66fc Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 29 Jul 2026 06:00:16 +0000 Subject: [PATCH 04/16] test: use MSI private IP for LocalDNS --- hack/e2e/infra/main.bicep | 1 + hack/e2e/lib/infra.sh | 8 +++++++- hack/e2e/lib/node-join-msi.sh | 4 +++- hack/e2e/lib/validate.sh | 4 +++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/hack/e2e/infra/main.bicep b/hack/e2e/infra/main.bicep index a07e5984..ec44699d 100644 --- a/hack/e2e/infra/main.bicep +++ b/hack/e2e/infra/main.bicep @@ -236,6 +236,7 @@ output clusterFqdn string = aksCluster.properties.fqdn output msiVmName string = vmMsi.outputs.vmName output msiVmIp string = vmMsi.outputs.publicIpAddress +output msiVmPrivateIp string = vmMsi.outputs.privateIpAddress output msiVmPrincipalId string = vmMsi.outputs.principalId output tokenVmName string = vmToken.outputs.vmName diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index cc3d457b..d7ba0224 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -107,7 +107,7 @@ infra_deploy() { --query properties.outputs \ -o json) - local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_principal_id + local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_private_ip msi_vm_principal_id local token_vm_name token_vm_ip token_vm_private_ip offline_vm_name offline_vm_ip offline_vm_private_ip local kubeadm_vm_name kubeadm_vm_ip admin_username @@ -115,6 +115,7 @@ infra_deploy() { cluster_id=$(echo "${outputs}" | jq -r '.clusterId.value') msi_vm_name=$(echo "${outputs}" | jq -r '.msiVmName.value') msi_vm_ip=$(echo "${outputs}" | jq -r '.msiVmIp.value') + msi_vm_private_ip=$(echo "${outputs}" | jq -r '.msiVmPrivateIp.value // ""') msi_vm_principal_id=$(echo "${outputs}" | jq -r '.msiVmPrincipalId.value') token_vm_name=$(echo "${outputs}" | jq -r '.tokenVmName.value') token_vm_ip=$(echo "${outputs}" | jq -r '.tokenVmIp.value') @@ -126,6 +127,10 @@ infra_deploy() { kubeadm_vm_ip=$(echo "${outputs}" | jq -r '.kubeadmVmIp.value') admin_username=$(echo "${outputs}" | jq -r '.adminUsername.value') + if [[ -z "${msi_vm_private_ip}" ]] || ! is_valid_ipv4 "${msi_vm_private_ip}"; then + log_error "Missing or invalid MSI VM private IP from deployment outputs: '${msi_vm_private_ip}'" + return 1 + fi if [[ -z "${token_vm_private_ip}" ]] || ! is_valid_ipv4 "${token_vm_private_ip}"; then log_error "Missing or invalid token VM private IP from deployment outputs: '${token_vm_private_ip}'" return 1 @@ -140,6 +145,7 @@ infra_deploy() { state_set "cluster_id" "${cluster_id}" state_set "msi_vm_name" "${msi_vm_name}" state_set "msi_vm_ip" "${msi_vm_ip}" + state_set "msi_vm_private_ip" "${msi_vm_private_ip}" state_set "msi_vm_principal_id" "${msi_vm_principal_id}" state_set "token_vm_name" "${token_vm_name}" state_set "token_vm_ip" "${token_vm_ip}" diff --git a/hack/e2e/lib/node-join-msi.sh b/hack/e2e/lib/node-join-msi.sh index 053d3593..b90892c0 100644 --- a/hack/e2e/lib/node-join-msi.sh +++ b/hack/e2e/lib/node-join-msi.sh @@ -24,6 +24,8 @@ node_join_msi() { local vm_ip vm_ip="$(state_get msi_vm_ip)" + local vm_private_ip + vm_private_ip="$(state_get msi_vm_private_ip)" local cluster_id cluster_id="$(state_get cluster_id)" local subscription_id @@ -56,7 +58,7 @@ node_join_msi() { "kubelet": { "clusterFQDN": "${server_url}", "caCertData": "${ca_cert_data}", - "nodeIP": "${vm_ip}" + "nodeIP": "${vm_private_ip}" } }, "agent": { diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 44ddd7a3..80686362 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -211,12 +211,13 @@ validate_all_nodes() { local msi_vm_name token_vm_name offline_vm_name kubeadm_vm_name local msi_vm_ip token_vm_ip offline_vm_ip kubeadm_vm_ip - local token_vm_private_ip offline_vm_private_ip + local msi_vm_private_ip token_vm_private_ip offline_vm_private_ip msi_vm_name="$(state_get msi_vm_name)" token_vm_name="$(state_get token_vm_name)" offline_vm_name="$(state_get offline_vm_name)" kubeadm_vm_name="$(state_get kubeadm_vm_name)" msi_vm_ip="$(state_get msi_vm_ip)" + msi_vm_private_ip="$(state_get msi_vm_private_ip)" token_vm_ip="$(state_get token_vm_ip)" offline_vm_ip="$(state_get offline_vm_ip)" kubeadm_vm_ip="$(state_get kubeadm_vm_ip)" @@ -228,6 +229,7 @@ validate_all_nodes() { validate_node_joined "${token_vm_name}" || failed=1 validate_node_joined "${offline_vm_name}" || failed=1 validate_node_joined "${kubeadm_vm_name}" || failed=1 + validate_node_ip "${msi_vm_name}" "${msi_vm_private_ip}" || failed=1 validate_node_ip "${token_vm_name}" "${token_vm_private_ip}" || failed=1 validate_node_ip "${offline_vm_name}" "${offline_vm_private_ip}" || failed=1 validate_npd_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 From 603813409ee56a4fcb7a29f18cf01b5f083543b7 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 30 Jul 2026 03:19:58 +0000 Subject: [PATCH 05/16] test: align LocalDNS behavior with AKS RP --- hack/e2e/lib/validate.sh | 49 +++++++++++++++++++++++++++-- pkg/config/adapter.go | 4 ++- pkg/config/localdns_adapter_test.go | 5 ++- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 80686362..af237e6b 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -170,7 +170,8 @@ REMOTE # --------------------------------------------------------------------------- # validate_localdns_status - Verify nspawn LocalDNS on the selected VM. validate_localdns_status() { - local vm_ip="$1" + local vm_name="$1" + local vm_ip="$2" log_info "Validating LocalDNS on ${vm_ip}..." remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' set -euo pipefail @@ -190,6 +191,50 @@ for chain in OUTPUT PREROUTING; do done done REMOTE + + local state_label + state_label="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/localdns-state}')" + if [[ "${state_label}" != "enabled" ]]; then + log_error "Node ${vm_name} localdns-state=${state_label}, expected enabled" + return 1 + fi + + local cluster_pod="localdns-clusterfirst-${vm_name}" + local default_pod="localdns-default-${vm_name}" + kubectl delete pod "${cluster_pod}" "${default_pod}" --ignore-not-found --wait=false >/dev/null + kubectl run "${cluster_pod}" --image=busybox:1.36 --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"${vm_name}\",\"dnsPolicy\":\"ClusterFirst\"}}" \ + --command -- nslookup kubernetes.default.svc.cluster.local >/dev/null + kubectl run "${default_pod}" --image=busybox:1.36 --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"${vm_name}\",\"dnsPolicy\":\"Default\"}}" \ + --command -- nslookup mcr.microsoft.com >/dev/null + + local pod + for pod in "${cluster_pod}" "${default_pod}"; do + local phase="" + for _ in $(seq 1 60); do + phase="$(kubectl get pod "${pod}" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + [[ "${phase}" == "Succeeded" || "${phase}" == "Failed" ]] && break + sleep 2 + done + if [[ "${phase}" != "Succeeded" ]]; then + kubectl describe pod "${pod}" >&2 || true + kubectl logs "${pod}" >&2 || true + log_error "LocalDNS query pod ${pod} ended in phase ${phase}" + return 1 + fi + done + + if ! kubectl logs "${cluster_pod}" | grep -Eq 'Server:[[:space:]]*169\.254\.10\.11'; then + log_error "ClusterFirst pod did not query LocalDNS cluster listener" + return 1 + fi + if ! kubectl logs "${default_pod}" | grep -Eq 'Server:[[:space:]]*169\.254\.10\.10'; then + log_error "Default-policy pod did not query LocalDNS node listener" + return 1 + fi + kubectl delete pod "${cluster_pod}" "${default_pod}" --ignore-not-found --wait=false >/dev/null + log_success "LocalDNS validation passed on ${vm_ip}" } @@ -233,7 +278,7 @@ validate_all_nodes() { validate_node_ip "${token_vm_name}" "${token_vm_private_ip}" || failed=1 validate_node_ip "${offline_vm_name}" "${offline_vm_private_ip}" || failed=1 validate_npd_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 - validate_localdns_status "${msi_vm_ip}" || failed=1 + validate_localdns_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 validate_npd_status "${token_vm_name}" "${token_vm_ip}" || failed=1 # TODO: re-enable once NPD is included in the upstream Unbounded bootstrap # artifact bundle and resolver used by offline artifact mode. diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index f9e16a30..0edb82ee 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -69,9 +69,11 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { Enabled: profile.Enabled(), CorefileTemplate: corefile, } + state := "disabled" if profile.Enabled() { - labels["kubernetes.azure.com/localdns"] = "enabled" + state = "enabled" } + labels["kubernetes.azure.com/localdns-state"] = state } if cfg.Bootstrap.OfflineArtifacts.Source != "" { diff --git a/pkg/config/localdns_adapter_test.go b/pkg/config/localdns_adapter_test.go index eba36356..20fbdb68 100644 --- a/pkg/config/localdns_adapter_test.go +++ b/pkg/config/localdns_adapter_test.go @@ -28,7 +28,7 @@ func TestToAgentConfigLocalDNS(t *testing.T) { if !strings.Contains(got.LocalDNS.CorefileTemplate, "{{ .ClusterDNSServiceIP }}") { t.Fatalf("CorefileTemplate missing cluster DNS placeholder:\n%s", got.LocalDNS.CorefileTemplate) } - if got.Kubelet.Labels["kubernetes.azure.com/localdns"] != "enabled" { + if got.Kubelet.Labels["kubernetes.azure.com/localdns-state"] != "enabled" { t.Fatalf("LocalDNS node label missing: %#v", got.Kubelet.Labels) } if got.Cluster.ClusterDNS != "10.0.0.10" { @@ -47,4 +47,7 @@ func TestToAgentConfigDisabledLocalDNS(t *testing.T) { if got.LocalDNS == nil || got.LocalDNS.Enabled { t.Fatalf("LocalDNS = %#v, want explicitly disabled", got.LocalDNS) } + if got.Kubelet.Labels["kubernetes.azure.com/localdns-state"] != "disabled" { + t.Fatalf("LocalDNS disabled node label missing: %#v", got.Kubelet.Labels) + } } From 4e0ba316a248f91fcc81c264431e827f0fd265ec Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 30 Jul 2026 19:06:35 +0000 Subject: [PATCH 06/16] refactor: isolate LocalDNS rendering --- pkg/config/config.go | 2 +- .../localdns/assets/localdns.corefile.tmpl | 25 ++ pkg/config/internal/localdns/localdns.go | 229 ++++++++++++++++ pkg/config/internal/localdns/localdns_test.go | 52 ++++ .../testdata/localdns.corefile.golden | 60 +++++ pkg/config/localdns.go | 249 +----------------- pkg/config/localdns_test.go | 2 +- 7 files changed, 376 insertions(+), 243 deletions(-) create mode 100644 pkg/config/internal/localdns/assets/localdns.corefile.tmpl create mode 100644 pkg/config/internal/localdns/localdns.go create mode 100644 pkg/config/internal/localdns/localdns_test.go create mode 100644 pkg/config/internal/localdns/testdata/localdns.corefile.golden diff --git a/pkg/config/config.go b/pkg/config/config.go index 98bdb546..07a8a6d1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -883,7 +883,7 @@ func (c *Config) validate() error { if err := c.Bootstrap.validate(); err != nil { return err } - if err := c.Networking.LocalDNS.validate(); err != nil { + if err := c.Networking.LocalDNS.Validate(); err != nil { return fmt.Errorf("invalid networking.localDNS: %w", err) } diff --git a/pkg/config/internal/localdns/assets/localdns.corefile.tmpl b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl new file mode 100644 index 00000000..cbb86f94 --- /dev/null +++ b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl @@ -0,0 +1,25 @@ +health-check.localdns.local:53 { + bind {{.NodeListener}} {{.ClusterListener}} + whoami +} + +{{range .Blocks}}{{.Zone}}:53 { + {{if .LogQueries}}log{{else}}errors{{end}} + bind {{.Listener}} + forward . {{.Upstream}} { + {{if .ForceTCP}}force_tcp + {{end}}policy {{.ForwardPolicy}} + max_concurrent {{.MaxConcurrent}} + } + ready {{.Listener}}:8181 + cache {{.CacheDuration}} { + success 9984 + denial 9984 + {{if .ServeStale}}serve_stale {{.ServeStaleDuration}}s {{.ServeStalePolicy}} + {{end}}servfail 0 + } + loop + prometheus {{$.MetricsAddress}} +} + +{{end}} \ No newline at end of file diff --git a/pkg/config/internal/localdns/localdns.go b/pkg/config/internal/localdns/localdns.go new file mode 100644 index 00000000..85dd1ce1 --- /dev/null +++ b/pkg/config/internal/localdns/localdns.go @@ -0,0 +1,229 @@ +package localdns + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "sort" + "strings" + "text/template" +) + +const ( + LocalDNSModeRequired = "Required" + LocalDNSModePreferred = "Preferred" + LocalDNSModeDisabled = "Disabled" +) + +// LocalDNSProfile mirrors the AKS node-pool LocalDNS configuration contract. +// Keep rendering behavior aligned with AgentBaker: +// https://github.com/Azure/AgentBaker/blob/62a783b7a967352bb4726e636d22930c9973f4f7/aks-node-controller/parser/templates/localdns.toml.gtpl +type LocalDNSProfile struct { + Mode string `json:"mode"` + VnetDNSOverrides map[string]LocalDNSOverride `json:"vnetDNSOverrides,omitempty"` + KubeDNSOverrides map[string]LocalDNSOverride `json:"kubeDNSOverrides,omitempty"` +} + +// LocalDNSOverride configures one CoreDNS server block. +type LocalDNSOverride struct { + QueryLogging string `json:"queryLogging,omitempty"` + Protocol string `json:"protocol,omitempty"` + ForwardDestination string `json:"forwardDestination,omitempty"` + ForwardPolicy string `json:"forwardPolicy,omitempty"` + MaxConcurrent int `json:"maxConcurrent,omitempty"` + CacheDurationInSeconds int `json:"cacheDurationInSeconds,omitempty"` + ServeStaleDurationInSeconds int `json:"serveStaleDurationInSeconds,omitempty"` + ServeStale string `json:"serveStale,omitempty"` +} + +// Validate verifies the AKS LocalDNS profile values accepted by the renderer. +func (p *LocalDNSProfile) Validate() error { + if p == nil { + return nil + } + if p.Mode != LocalDNSModeRequired && p.Mode != LocalDNSModePreferred && p.Mode != LocalDNSModeDisabled { + return fmt.Errorf("mode must be Required, Preferred, or Disabled") + } + var errs []error + for class, overrides := range map[string]map[string]LocalDNSOverride{ + "vnetDNSOverrides": p.VnetDNSOverrides, + "kubeDNSOverrides": p.KubeDNSOverrides, + } { + for zone, override := range overrides { + if strings.TrimSpace(zone) == "" || strings.ContainsAny(zone, "{} \t\r\n") { + errs = append(errs, fmt.Errorf("%s zone %q is invalid", class, zone)) + } + if err := override.validate(); err != nil { + errs = append(errs, fmt.Errorf("%s[%q]: %w", class, zone, err)) + } + } + } + return errors.Join(errs...) +} + +func (o LocalDNSOverride) validate() error { + var errs []error + if o.QueryLogging != "" && o.QueryLogging != "Error" && o.QueryLogging != "Log" { + errs = append(errs, fmt.Errorf("queryLogging must be Error or Log")) + } + if o.Protocol != "" && o.Protocol != "PreferUDP" && o.Protocol != "ForceTCP" { + errs = append(errs, fmt.Errorf("protocol must be PreferUDP or ForceTCP")) + } + if o.ForwardDestination != "" && o.ForwardDestination != "VnetDNS" && o.ForwardDestination != "ClusterCoreDNS" { + errs = append(errs, fmt.Errorf("forwardDestination must be VnetDNS or ClusterCoreDNS")) + } + if o.ForwardPolicy != "" && o.ForwardPolicy != "Sequential" && o.ForwardPolicy != "RoundRobin" && o.ForwardPolicy != "Random" { + errs = append(errs, fmt.Errorf("forwardPolicy must be Sequential, RoundRobin, or Random")) + } + if o.ServeStale != "" && o.ServeStale != "Disable" && o.ServeStale != "Verify" && o.ServeStale != "Immediate" { + errs = append(errs, fmt.Errorf("serveStale must be Disable, Verify, or Immediate")) + } + for name, value := range map[string]int{ + "maxConcurrent": o.MaxConcurrent, + "cacheDurationInSeconds": o.CacheDurationInSeconds, + "serveStaleDurationInSeconds": o.ServeStaleDurationInSeconds, + } { + if value < 0 { + errs = append(errs, fmt.Errorf("%s must not be negative", name)) + } + } + return errors.Join(errs...) +} + +// Enabled reports whether the profile requires LocalDNS installation. +func (p *LocalDNSProfile) Enabled() bool { + return p != nil && p.Mode == LocalDNSModeRequired +} + +//go:embed assets/localdns.corefile.tmpl +var aksLocalDNSCorefileTemplate string + +type localDNSCorefileData struct { + NodeListener string + ClusterListener string + MetricsAddress string + Blocks []localDNSCorefileBlock +} + +type localDNSCorefileBlock struct { + Zone string + Listener string + Upstream string + LogQueries bool + ForceTCP bool + ForwardPolicy string + MaxConcurrent int + CacheDuration int + ServeStale bool + ServeStaleDuration int + ServeStalePolicy string +} + +// CorefileTemplate renders AKS LocalDNS policy into an Unbounded Corefile template. +func (p *LocalDNSProfile) CorefileTemplate() (string, error) { + if p == nil { + return "", nil + } + if err := p.Validate(); err != nil { + return "", err + } + if !p.Enabled() { + return "", nil + } + + vnet := p.VnetDNSOverrides + if len(vnet) == 0 { + vnet = map[string]LocalDNSOverride{".": {ForwardDestination: "VnetDNS"}} + } + kube := p.KubeDNSOverrides + if len(kube) == 0 { + kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} + } + + data := localDNSCorefileData{ + NodeListener: "{{ .NodeListenerIP }}", + ClusterListener: "{{ .ClusterListenerIP }}", + MetricsAddress: "{{ .MetricsAddress }}", + Blocks: append(localDNSCorefileBlocks(vnet, "{{ .NodeListenerIP }}", "VnetDNS"), localDNSCorefileBlocks(kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS")...), + } + tmpl, err := template.New("aks-localdns-corefile").Parse(aksLocalDNSCorefileTemplate) + if err != nil { + return "", fmt.Errorf("parse AKS LocalDNS Corefile template: %w", err) + } + var out bytes.Buffer + if err := tmpl.Execute(&out, data); err != nil { + return "", fmt.Errorf("render AKS LocalDNS Corefile template: %w", err) + } + return out.String(), nil +} + +func localDNSCorefileBlocks(overrides map[string]LocalDNSOverride, listener, defaultDestination string) []localDNSCorefileBlock { + zones := make([]string, 0, len(overrides)) + for zone := range overrides { + zones = append(zones, zone) + } + sort.Strings(zones) + + blocks := make([]localDNSCorefileBlock, 0, len(zones)) + for _, zone := range zones { + o := withLocalDNSDefaults(overrides[zone], defaultDestination) + upstream := "{{ .NodeUpstreamIPsJoined }}" + if o.ForwardDestination == "ClusterCoreDNS" { + upstream = "{{ .ClusterDNSServiceIP }}" + } + blocks = append(blocks, localDNSCorefileBlock{ + Zone: zone, + Listener: listener, + Upstream: upstream, + LogQueries: o.QueryLogging == "Log", + ForceTCP: o.Protocol == "ForceTCP", + ForwardPolicy: localDNSForwardPolicy(o.ForwardPolicy), + MaxConcurrent: o.MaxConcurrent, + CacheDuration: o.CacheDurationInSeconds, + ServeStale: o.ServeStale != "Disable", + ServeStaleDuration: o.ServeStaleDurationInSeconds, + ServeStalePolicy: strings.ToLower(o.ServeStale), + }) + } + return blocks +} + +func withLocalDNSDefaults(o LocalDNSOverride, destination string) LocalDNSOverride { + if o.QueryLogging == "" { + o.QueryLogging = "Error" + } + if o.Protocol == "" { + o.Protocol = "ForceTCP" + } + if o.ForwardDestination == "" { + o.ForwardDestination = destination + } + if o.ForwardPolicy == "" { + o.ForwardPolicy = "Sequential" + } + if o.MaxConcurrent == 0 { + o.MaxConcurrent = 1000 + } + if o.CacheDurationInSeconds == 0 { + o.CacheDurationInSeconds = 3600 + } + if o.ServeStaleDurationInSeconds == 0 { + o.ServeStaleDurationInSeconds = 3600 + } + if o.ServeStale == "" { + o.ServeStale = "Immediate" + } + return o +} + +func localDNSForwardPolicy(policy string) string { + switch policy { + case "RoundRobin": + return "round_robin" + case "Random": + return "random" + default: + return "sequential" + } +} diff --git a/pkg/config/internal/localdns/localdns_test.go b/pkg/config/internal/localdns/localdns_test.go new file mode 100644 index 00000000..536f1e85 --- /dev/null +++ b/pkg/config/internal/localdns/localdns_test.go @@ -0,0 +1,52 @@ +package localdns + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCorefileTemplateGolden(t *testing.T) { + t.Parallel() + + profile := &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + VnetDNSOverrides: map[string]LocalDNSOverride{ + ".": { + QueryLogging: "Log", Protocol: "PreferUDP", ForwardDestination: "VnetDNS", + ForwardPolicy: "Sequential", MaxConcurrent: 1000, CacheDurationInSeconds: 30, + ServeStaleDurationInSeconds: 60, ServeStale: "Immediate", + }, + "cluster.local": { + QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", + ForwardPolicy: "RoundRobin", MaxConcurrent: 2000, CacheDurationInSeconds: 45, + ServeStale: "Disable", + }, + }, + KubeDNSOverrides: map[string]LocalDNSOverride{ + ".": { + QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", + ForwardPolicy: "Random", MaxConcurrent: 1500, CacheDurationInSeconds: 90, + ServeStaleDurationInSeconds: 120, ServeStale: "Verify", + }, + }, + } + + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatalf("CorefileTemplate() error = %v", err) + } + goldenPath := filepath.Join("testdata", "localdns.corefile.golden") + if os.Getenv("UPDATE_GOLDEN") == "1" { + if err := os.WriteFile(goldenPath, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(goldenPath) + if err != nil { + t.Fatal(err) + } + if got != string(want) { + t.Fatalf("Corefile mismatch (-want +got):\n--- want ---\n%s\n--- got ---\n%s", want, got) + } +} diff --git a/pkg/config/internal/localdns/testdata/localdns.corefile.golden b/pkg/config/internal/localdns/testdata/localdns.corefile.golden new file mode 100644 index 00000000..c1dbcd8d --- /dev/null +++ b/pkg/config/internal/localdns/testdata/localdns.corefile.golden @@ -0,0 +1,60 @@ +health-check.localdns.local:53 { + bind {{ .NodeListenerIP }} {{ .ClusterListenerIP }} + whoami +} + +.:53 { + log + bind {{ .NodeListenerIP }} + forward . {{ .NodeUpstreamIPsJoined }} { + policy sequential + max_concurrent 1000 + } + ready {{ .NodeListenerIP }}:8181 + cache 30 { + success 9984 + denial 9984 + serve_stale 60s immediate + servfail 0 + } + loop + prometheus {{ .MetricsAddress }} +} + +cluster.local:53 { + errors + bind {{ .NodeListenerIP }} + forward . {{ .ClusterDNSServiceIP }} { + force_tcp + policy round_robin + max_concurrent 2000 + } + ready {{ .NodeListenerIP }}:8181 + cache 45 { + success 9984 + denial 9984 + servfail 0 + } + loop + prometheus {{ .MetricsAddress }} +} + +.:53 { + errors + bind {{ .ClusterListenerIP }} + forward . {{ .ClusterDNSServiceIP }} { + force_tcp + policy random + max_concurrent 1500 + } + ready {{ .ClusterListenerIP }}:8181 + cache 90 { + success 9984 + denial 9984 + serve_stale 120s verify + servfail 0 + } + loop + prometheus {{ .MetricsAddress }} +} + diff --git a/pkg/config/localdns.go b/pkg/config/localdns.go index 4f487d5c..0422fbfe 100644 --- a/pkg/config/localdns.go +++ b/pkg/config/localdns.go @@ -1,248 +1,15 @@ package config -import ( - "bytes" - "errors" - "fmt" - "sort" - "strings" - "text/template" -) +import "github.com/Azure/AKSFlexNode/pkg/config/internal/localdns" const ( - LocalDNSModeRequired = "Required" - LocalDNSModePreferred = "Preferred" - LocalDNSModeDisabled = "Disabled" + LocalDNSModeRequired = localdns.LocalDNSModeRequired + LocalDNSModePreferred = localdns.LocalDNSModePreferred + LocalDNSModeDisabled = localdns.LocalDNSModeDisabled ) -// LocalDNSProfile mirrors the AKS node-pool LocalDNS configuration contract. -type LocalDNSProfile struct { - Mode string `json:"mode"` - VnetDNSOverrides map[string]LocalDNSOverride `json:"vnetDNSOverrides,omitempty"` - KubeDNSOverrides map[string]LocalDNSOverride `json:"kubeDNSOverrides,omitempty"` -} - -// LocalDNSOverride configures one CoreDNS server block. -type LocalDNSOverride struct { - QueryLogging string `json:"queryLogging,omitempty"` - Protocol string `json:"protocol,omitempty"` - ForwardDestination string `json:"forwardDestination,omitempty"` - ForwardPolicy string `json:"forwardPolicy,omitempty"` - MaxConcurrent int `json:"maxConcurrent,omitempty"` - CacheDurationInSeconds int `json:"cacheDurationInSeconds,omitempty"` - ServeStaleDurationInSeconds int `json:"serveStaleDurationInSeconds,omitempty"` - ServeStale string `json:"serveStale,omitempty"` -} - -func (p *LocalDNSProfile) validate() error { - if p == nil { - return nil - } - if p.Mode != LocalDNSModeRequired && p.Mode != LocalDNSModePreferred && p.Mode != LocalDNSModeDisabled { - return fmt.Errorf("mode must be Required, Preferred, or Disabled") - } - var errs []error - for class, overrides := range map[string]map[string]LocalDNSOverride{ - "vnetDNSOverrides": p.VnetDNSOverrides, - "kubeDNSOverrides": p.KubeDNSOverrides, - } { - for zone, override := range overrides { - if strings.TrimSpace(zone) == "" || strings.ContainsAny(zone, "{} \t\r\n") { - errs = append(errs, fmt.Errorf("%s zone %q is invalid", class, zone)) - } - if err := override.validate(); err != nil { - errs = append(errs, fmt.Errorf("%s[%q]: %w", class, zone, err)) - } - } - } - return errors.Join(errs...) -} - -func (o LocalDNSOverride) validate() error { - var errs []error - if o.QueryLogging != "" && o.QueryLogging != "Error" && o.QueryLogging != "Log" { - errs = append(errs, fmt.Errorf("queryLogging must be Error or Log")) - } - if o.Protocol != "" && o.Protocol != "PreferUDP" && o.Protocol != "ForceTCP" { - errs = append(errs, fmt.Errorf("protocol must be PreferUDP or ForceTCP")) - } - if o.ForwardDestination != "" && o.ForwardDestination != "VnetDNS" && o.ForwardDestination != "ClusterCoreDNS" { - errs = append(errs, fmt.Errorf("forwardDestination must be VnetDNS or ClusterCoreDNS")) - } - if o.ForwardPolicy != "" && o.ForwardPolicy != "Sequential" && o.ForwardPolicy != "RoundRobin" && o.ForwardPolicy != "Random" { - errs = append(errs, fmt.Errorf("forwardPolicy must be Sequential, RoundRobin, or Random")) - } - if o.ServeStale != "" && o.ServeStale != "Disable" && o.ServeStale != "Verify" && o.ServeStale != "Immediate" { - errs = append(errs, fmt.Errorf("serveStale must be Disable, Verify, or Immediate")) - } - for name, value := range map[string]int{ - "maxConcurrent": o.MaxConcurrent, - "cacheDurationInSeconds": o.CacheDurationInSeconds, - "serveStaleDurationInSeconds": o.ServeStaleDurationInSeconds, - } { - if value < 0 { - errs = append(errs, fmt.Errorf("%s must not be negative", name)) - } - } - return errors.Join(errs...) -} - -// Enabled reports whether the profile requires LocalDNS installation. -func (p *LocalDNSProfile) Enabled() bool { - return p != nil && p.Mode == LocalDNSModeRequired -} - -const aksLocalDNSCorefileTemplate = `health-check.localdns.local:53 { - bind {{.NodeListener}} {{.ClusterListener}} - whoami -} - -{{range .Blocks}}{{.Zone}}:53 { - {{if .LogQueries}}log{{else}}errors{{end}} - bind {{.Listener}} - forward . {{.Upstream}} { - {{if .ForceTCP}}force_tcp - {{end}}policy {{.ForwardPolicy}} - max_concurrent {{.MaxConcurrent}} - } - ready {{.Listener}}:8181 - cache {{.CacheDuration}} { - success 9984 - denial 9984 - {{if .ServeStale}}serve_stale {{.ServeStaleDuration}}s {{.ServeStalePolicy}} - {{end}}servfail 0 - } - loop - prometheus {{$.MetricsAddress}} -} - -{{end}}` - -type localDNSCorefileData struct { - NodeListener string - ClusterListener string - MetricsAddress string - Blocks []localDNSCorefileBlock -} - -type localDNSCorefileBlock struct { - Zone string - Listener string - Upstream string - LogQueries bool - ForceTCP bool - ForwardPolicy string - MaxConcurrent int - CacheDuration int - ServeStale bool - ServeStaleDuration int - ServeStalePolicy string -} - -// CorefileTemplate renders AKS LocalDNS policy into an Unbounded Corefile template. -func (p *LocalDNSProfile) CorefileTemplate() (string, error) { - if p == nil { - return "", nil - } - if err := p.validate(); err != nil { - return "", err - } - if !p.Enabled() { - return "", nil - } - - vnet := p.VnetDNSOverrides - if len(vnet) == 0 { - vnet = map[string]LocalDNSOverride{".": {ForwardDestination: "VnetDNS"}} - } - kube := p.KubeDNSOverrides - if len(kube) == 0 { - kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} - } - - data := localDNSCorefileData{ - NodeListener: "{{ .NodeListenerIP }}", - ClusterListener: "{{ .ClusterListenerIP }}", - MetricsAddress: "{{ .MetricsAddress }}", - Blocks: append(localDNSCorefileBlocks(vnet, "{{ .NodeListenerIP }}", "VnetDNS"), localDNSCorefileBlocks(kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS")...), - } - tmpl, err := template.New("aks-localdns-corefile").Parse(aksLocalDNSCorefileTemplate) - if err != nil { - return "", fmt.Errorf("parse AKS LocalDNS Corefile template: %w", err) - } - var out bytes.Buffer - if err := tmpl.Execute(&out, data); err != nil { - return "", fmt.Errorf("render AKS LocalDNS Corefile template: %w", err) - } - return out.String(), nil -} - -func localDNSCorefileBlocks(overrides map[string]LocalDNSOverride, listener, defaultDestination string) []localDNSCorefileBlock { - zones := make([]string, 0, len(overrides)) - for zone := range overrides { - zones = append(zones, zone) - } - sort.Strings(zones) - - blocks := make([]localDNSCorefileBlock, 0, len(zones)) - for _, zone := range zones { - o := withLocalDNSDefaults(overrides[zone], defaultDestination) - upstream := "{{ .NodeUpstreamIPsJoined }}" - if o.ForwardDestination == "ClusterCoreDNS" { - upstream = "{{ .ClusterDNSServiceIP }}" - } - blocks = append(blocks, localDNSCorefileBlock{ - Zone: zone, - Listener: listener, - Upstream: upstream, - LogQueries: o.QueryLogging == "Log", - ForceTCP: o.Protocol == "ForceTCP", - ForwardPolicy: localDNSForwardPolicy(o.ForwardPolicy), - MaxConcurrent: o.MaxConcurrent, - CacheDuration: o.CacheDurationInSeconds, - ServeStale: o.ServeStale != "Disable", - ServeStaleDuration: o.ServeStaleDurationInSeconds, - ServeStalePolicy: strings.ToLower(o.ServeStale), - }) - } - return blocks -} - -func withLocalDNSDefaults(o LocalDNSOverride, destination string) LocalDNSOverride { - if o.QueryLogging == "" { - o.QueryLogging = "Error" - } - if o.Protocol == "" { - o.Protocol = "ForceTCP" - } - if o.ForwardDestination == "" { - o.ForwardDestination = destination - } - if o.ForwardPolicy == "" { - o.ForwardPolicy = "Sequential" - } - if o.MaxConcurrent == 0 { - o.MaxConcurrent = 1000 - } - if o.CacheDurationInSeconds == 0 { - o.CacheDurationInSeconds = 3600 - } - if o.ServeStaleDurationInSeconds == 0 { - o.ServeStaleDurationInSeconds = 3600 - } - if o.ServeStale == "" { - o.ServeStale = "Immediate" - } - return o -} +// LocalDNSProfile is the AKS node-pool LocalDNS configuration contract. +type LocalDNSProfile = localdns.LocalDNSProfile -func localDNSForwardPolicy(policy string) string { - switch policy { - case "RoundRobin": - return "round_robin" - case "Random": - return "random" - default: - return "sequential" - } -} +// LocalDNSOverride configures one AKS LocalDNS server block. +type LocalDNSOverride = localdns.LocalDNSOverride diff --git a/pkg/config/localdns_test.go b/pkg/config/localdns_test.go index 7687ec89..81524ed1 100644 --- a/pkg/config/localdns_test.go +++ b/pkg/config/localdns_test.go @@ -131,7 +131,7 @@ func TestLocalDNSProfileValidation(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - err := test.profile.validate() + err := test.profile.Validate() if test.wantErr == "" && err != nil { t.Fatalf("validate() error = %v", err) } From 20eb62a1647eb491377d33d53a614cd479b55658 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 30 Jul 2026 19:44:16 +0000 Subject: [PATCH 07/16] test: validate LocalDNS disable through repave --- hack/e2e/lib/upgrade-drift.sh | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/hack/e2e/lib/upgrade-drift.sh b/hack/e2e/lib/upgrade-drift.sh index f1be588f..c2c3970a 100644 --- a/hack/e2e/lib/upgrade-drift.sh +++ b/hack/e2e/lib/upgrade-drift.sh @@ -209,11 +209,60 @@ upgrade_drift_mode() { log_success "${mode} controller machine repave passed in $(timer_elapsed "${start}")s" } +localdns_disable_repave_msi() { + log_section "LocalDNS Disable Repave (MSI node)" + local desired_version settings_version vm_name vm_ip snapshot old_active_machine old_state old_version old_settings_version old_node_uid + desired_version="$(_cluster_current_kubernetes_version)" + settings_version="localdns-disabled-$(date +%s)" + vm_name="$(_mode_vm_name msi)" + vm_ip="$(_mode_vm_ip msi)" + + snapshot="$(_remote_active_machine_snapshot "${vm_ip}")" + IFS='|' read -r old_active_machine old_state old_version old_settings_version <<<"${snapshot}" + old_node_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" + + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail +config=/etc/aks-flex-node/config.json +tmp=$(mktemp /etc/aks-flex-node/config.json.XXXXXX) +jq '.networking.localDNS.mode = "Disabled"' "${config}" > "${tmp}" +chmod 0600 "${tmp}" +mv "${tmp}" "${config}" +systemctl restart aks-flex-node-agent.service +systemctl is-active --quiet aks-flex-node-agent.service +REMOTE + + _trigger_mode_repave msi "${desired_version}" "${settings_version}" + _wait_for_mode_repave msi "${desired_version}" "${settings_version}" "${old_active_machine}" "${old_node_uid}" + + local state_label cluster_dns + state_label="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/localdns-state}')" + cluster_dns="$(kubectl -n kube-system get service kube-dns -o jsonpath='{.spec.clusterIP}')" + if [[ "${state_label}" != "disabled" ]]; then + log_error "Node ${vm_name} localdns-state=${state_label}, expected disabled" + return 1 + fi + + remote_exec "${vm_ip}" "CLUSTER_DNS=${cluster_dns} sudo -E bash -s" <<'REMOTE' +set -euo pipefail +machine=$(machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}') +test -n "${machine}" +! systemd-run --quiet --pipe --wait --machine="${machine}" systemctl cat localdns.service >/dev/null 2>&1 +! ip link show localdns >/dev/null 2>&1 +! iptables -w -t raw -S | grep -q 'unbounded-localdns: skip conntrack' +systemd-run --quiet --pipe --wait --machine="${machine}" \ + grep -q -- "--cluster-dns=${CLUSTER_DNS}" /etc/systemd/system/kubelet.service.d/20-node-config.conf +REMOTE + + log_success "MSI LocalDNS disable-through-repave validation passed" +} + upgrade_drift_all() { log_section "Controller Machine Repave (all modes)" upgrade_drift_mode msi upgrade_drift_mode token upgrade_drift_mode kubeadm + localdns_disable_repave_msi } upgrade_drift_msi() { upgrade_drift_mode msi; } From e6d87dbac24e6a1bb311fac5c3395d825c3df913 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 4 Aug 2026 20:53:46 +0000 Subject: [PATCH 08/16] build: bump Unbounded for nspawn LocalDNS --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f7009d23..56079775 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.2 + github.com/Azure/unbounded v0.2.3-0.20260804212734-7fc47a19d0c3 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index dd35efd4..3d70b02f 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.2 h1:uu5hYj20UBbrSAlf4qnDiMTGaR0xxsYnkdbTWTdBHX8= -github.com/Azure/unbounded v0.2.2/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw= +github.com/Azure/unbounded v0.2.3-0.20260804212734-7fc47a19d0c3 h1:QNdVeXxDMQJaVArB3cRuuZY5qd2Ww13EudPaZ3AUyf0= +github.com/Azure/unbounded v0.2.3-0.20260804212734-7fc47a19d0c3/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 9e77ac8ce7fca641a27ee2ad8564463f0ce56ed1 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 4 Aug 2026 21:49:40 +0000 Subject: [PATCH 09/16] test: prepare LocalDNS E2E host resolver --- hack/e2e/lib/node-join-msi.sh | 62 ++++++++++++++++++++++++++++++++++- hack/e2e/lib/validate.sh | 14 +++++--- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/hack/e2e/lib/node-join-msi.sh b/hack/e2e/lib/node-join-msi.sh index b90892c0..6a94715e 100644 --- a/hack/e2e/lib/node-join-msi.sh +++ b/hack/e2e/lib/node-join-msi.sh @@ -14,6 +14,63 @@ readonly _E2E_NODE_JOIN_MSI_LOADED=1 # shellcheck disable=SC1091 source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +# --------------------------------------------------------------------------- +# prepare_localdns_host_resolver - Remove DHCP search/routing domains. +# +# Unbounded intentionally rejects systemd-resolved split-DNS layouts because +# flattening per-domain routing into one LocalDNS upstream list changes DNS +# semantics. Azure's Ubuntu image supplies a DHCP search domain by default, so +# put the disposable E2E host into the supported single-upstream layout before +# LocalDNS preflight runs. +# --------------------------------------------------------------------------- +prepare_localdns_host_resolver() { + local vm_ip="$1" + + log_info "Configuring the MSI host resolver for LocalDNS..." + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail + +interface="$(ip -4 route show default | awk 'NR == 1 { print $5 }')" +if [[ ! "${interface}" =~ ^[a-zA-Z0-9_.:-]+$ ]]; then + echo "could not determine a safe primary interface name: ${interface}" >&2 + exit 1 +fi + +cat > /etc/netplan/99-aks-flex-localdns.yaml <&2 + resolvectl domain >&2 + exit 1 +fi + +resolvectl domain +resolvectl dns "${interface}" +REMOTE +} + # --------------------------------------------------------------------------- # node_join_msi - Join the MSI VM # --------------------------------------------------------------------------- @@ -107,7 +164,10 @@ node_join_msi() { } EOF - # Step 2: Publish the AKS Machine goal and deploy the agent. + # Step 2: Put systemd-resolved into the layout supported by LocalDNS. + prepare_localdns_host_resolver "${vm_ip}" + + # Step 3: Publish the AKS Machine goal and deploy the agent. ensure_flex_controller machine_configmap_upsert "$(state_get msi_vm_name)" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}" _deploy_and_start_agent "${vm_ip}" "${config_file}" "aks-flex-node-msi" diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index af237e6b..ac299d65 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -178,15 +178,19 @@ set -euo pipefail machine=$(sudo machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}') test -n "${machine}" sudo systemd-run --quiet --pipe --wait --machine="${machine}" systemctl is-active --quiet localdns.service -sudo systemd-run --quiet --pipe --wait --machine="${machine}" grep -qx 'nameserver 169.254.10.10' /etc/resolv.conf +sudo systemd-run --quiet --pipe --wait --machine="${machine}" \ + grep -qx 'nameserver 169.254.10.10' /etc/unbounded/localdns/resolv.conf +sudo systemd-run --quiet --pipe --wait --machine="${machine}" \ + systemctl cat kubelet.service | grep -q -- '--resolv-conf=/etc/unbounded/localdns/resolv.conf' sudo ip address show dev localdns | grep -q '169.254.10.10/32' sudo ip address show dev localdns | grep -q '169.254.10.11/32' -for chain in OUTPUT PREROUTING; do +for chain in output prerouting; do + rules="$(sudo nft list chain ip unbounded_localdns "${chain}")" for address in 169.254.10.10 169.254.10.11; do for protocol in tcp udp; do - sudo iptables -w -t raw -C "${chain}" -m comment \ - --comment 'unbounded-localdns: skip conntrack' \ - -p "${protocol}" -d "${address}" --dport 53 -j NOTRACK + grep -Fq \ + "ip daddr ${address} ${protocol} dport 53 notrack comment \"unbounded-localdns: skip conntrack\"" \ + <<<"${rules}" done done done From 74646a3218f1532398f867e7aa48401b89dd4fc5 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 4 Aug 2026 22:58:16 +0000 Subject: [PATCH 10/16] feat: configure kubelet through generated config --- docs/usages/configuration.md | 39 ++++++++++++++ go.mod | 12 ++--- go.sum | 24 ++++----- pkg/config/adapter.go | 19 +++++++ pkg/config/adapter_test.go | 43 +++++++++++++-- pkg/config/config.go | 28 ++++++++++ pkg/config/copy_test.go | 14 +++++ pkg/config/kubelet_config_test.go | 82 +++++++++++++++++++++++++++++ pkg/daemon/nodeoperator.go | 30 +++++++++-- pkg/daemon/nodeoperator_test.go | 87 +++++++++++++++++++++++++++++++ 10 files changed, 352 insertions(+), 26 deletions(-) create mode 100644 pkg/config/kubelet_config_test.go diff --git a/docs/usages/configuration.md b/docs/usages/configuration.md index 11d8084e..ccd992aa 100644 --- a/docs/usages/configuration.md +++ b/docs/usages/configuration.md @@ -142,6 +142,12 @@ At least one join or Azure authentication method must be configured. `azure.boot | `node.kubelet.clusterFQDN` | string | Kubernetes API server FQDN. Required for bootstrap token mode. | `example.hcp.canadacentral.azmk8s.io` | | `node.kubelet.caCertData` | string | Base64-encoded cluster CA data. Required for bootstrap token mode. | `` | | `node.kubelet.nodeIP` | string | Optional node IP override for kubelet `--node-ip`. | `10.0.0.4` | +| `node.kubelet.imageCredentialProvider.configPath` | string | Optional absolute path inside the nspawn machine to a kubelet exec image credential provider configuration file or supported configuration directory. Must be set with `binDir`. | `/etc/kubernetes/credential-provider.yaml` | +| `node.kubelet.imageCredentialProvider.binDir` | string | Optional absolute path inside the nspawn machine containing exec image credential provider binaries. Must be set with `configPath`. | `/usr/local/lib/kubelet-credential-providers` | + +Provider paths must be clean absolute machine paths without whitespace or systemd argument characters. Include the provider files in the OCI rootfs or expose them with read-only `bootstrap.additionalHostMounts`. + +The image credential provider executes a plugin to obtain short-lived pull credentials; it does not place registry passwords or tokens in the FlexNode configuration. Do not store static registry credentials in this file or provider configuration. ## Component Versions @@ -348,3 +354,36 @@ Use `bootstrap.additionalHostMounts` to expose host files or directories inside ``` Read-only entries render as systemd-nspawn `BindReadOnly=` directives; writable entries render as `Bind=` directives. + +### Image Pull Credential Provider + +Kubelet exec image credential providers can obtain short-lived registry credentials without storing tokens in the FlexNode config. The provider configuration and executable must exist inside the nspawn machine, either in the OCI rootfs or through host mounts: + +```json +{ + "bootstrap": { + "additionalHostMounts": [ + { + "source": "/opt/aks-flex-node/credential-provider/config.yaml", + "target": "/etc/kubernetes/credential-provider.yaml", + "readOnly": true + }, + { + "source": "/opt/aks-flex-node/credential-provider/bin", + "target": "/usr/local/lib/kubelet-credential-providers", + "readOnly": true + } + ] + }, + "node": { + "kubelet": { + "imageCredentialProvider": { + "configPath": "/etc/kubernetes/credential-provider.yaml", + "binDir": "/usr/local/lib/kubelet-credential-providers" + } + } + } +} +``` + +Provider binaries must be executable before the machine starts. Mount provider assets read-only unless the provider explicitly requires writable state. diff --git a/go.mod b/go.mod index f7009d23..8360aded 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,14 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.2 + github.com/Azure/unbounded v0.2.3-0.20260804222626-7dacc2cdd8ac github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.2 - k8s.io/api v0.36.2 - k8s.io/apimachinery v0.36.2 - k8s.io/client-go v0.36.2 + k8s.io/api v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/client-go v0.36.3 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 ) @@ -101,12 +101,12 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.36.0 // indirect + k8s.io/apiextensions-apiserver v0.36.3 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 // indirect oras.land/oras-go/v2 v2.6.2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index dd35efd4..3846ef3a 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.2 h1:uu5hYj20UBbrSAlf4qnDiMTGaR0xxsYnkdbTWTdBHX8= -github.com/Azure/unbounded v0.2.2/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw= +github.com/Azure/unbounded v0.2.3-0.20260804222626-7dacc2cdd8ac h1:duo0qxzi02CHxwaXo+btkls92ZBCDV6BfyG9UgqyO+A= +github.com/Azure/unbounded v0.2.3-0.20260804222626-7dacc2cdd8ac/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= @@ -348,14 +348,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= -k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= -k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= -k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= -k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= -k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= -k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= -k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= +k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0= +k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 h1:Sztf7ESG9tAXRW/ACJZjrj5jhdOUqS2KFRQT+CTvu78= @@ -370,7 +370,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index dfd302b3..5ec92755 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -42,6 +42,7 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { NodeIP: cfg.Node.Kubelet.NodeIP, Labels: cfg.Node.Labels, RegisterWithTaints: cfg.Node.Taints, + Configuration: kubeletConfiguration(cfg), }, CRI: agentconfig.CRIConfig{ Containerd: agentconfig.ContainerdConfig{ @@ -57,6 +58,13 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { }, } + if provider := cfg.Node.Kubelet.ImageCredentialProvider; provider != nil { + ac.Kubelet.ImageCredentialProvider = &agentconfig.ImageCredentialProvider{ + ConfigPath: provider.ConfigPath, + BinDir: provider.BinDir, + } + } + if cfg.Bootstrap.OfflineArtifacts.Source != "" { ac.OfflineArtifacts = &agentconfig.AgentOfflineArtifacts{ Source: cfg.Bootstrap.OfflineArtifacts.Source, @@ -103,6 +111,17 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { return ac } +func kubeletConfiguration(cfg *Config) map[string]any { + return map[string]any{ + "maxPods": cfg.Node.MaxPods, + "imageGCHighThresholdPercent": cfg.Node.Kubelet.ImageGCHighThreshold, + "imageGCLowThresholdPercent": cfg.Node.Kubelet.ImageGCLowThreshold, + "logging": map[string]any{ + "verbosity": cfg.Node.Kubelet.Verbosity, + }, + } +} + // ResolveMachineGoalState converts FlexNode config to the shared agent config // and resolves the nspawn machine goal state. Bootstrap and preflight both use // this helper so preflight validates the same sources that bootstrap consumes. diff --git a/pkg/config/adapter_test.go b/pkg/config/adapter_test.go index 510af5e4..82bdf6e5 100644 --- a/pkg/config/adapter_test.go +++ b/pkg/config/adapter_test.go @@ -19,12 +19,20 @@ func TestToAgentConfig_BootstrapToken(t *testing.T) { Components: ComponentsConfig{Kubernetes: "1.30.0"}, Networking: NetworkingConfig{DNSServiceIP: "10.0.0.10"}, Node: NodeConfig{ - Labels: map[string]string{"env": "test"}, - Taints: []string{"dedicated=infra:NoSchedule"}, + MaxPods: 42, + Labels: map[string]string{"env": "test"}, + Taints: []string{"dedicated=infra:NoSchedule"}, Kubelet: KubeletConfig{ - ClusterFQDN: "api.example.com:6443", - CACertData: "dGVzdC1jYS1kYXRh", - NodeIP: "10.225.0.4", + Verbosity: 4, + ImageGCHighThreshold: 90, + ImageGCLowThreshold: 75, + ClusterFQDN: "api.example.com:6443", + CACertData: "dGVzdC1jYS1kYXRh", + NodeIP: "10.225.0.4", + ImageCredentialProvider: &ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, }, }, } @@ -65,6 +73,31 @@ func TestToAgentConfig_BootstrapToken(t *testing.T) { if len(ac.Kubelet.RegisterWithTaints) != 1 || ac.Kubelet.RegisterWithTaints[0] != "dedicated=infra:NoSchedule" { t.Fatalf("Kubelet.RegisterWithTaints=%v, want [dedicated=infra:NoSchedule]", ac.Kubelet.RegisterWithTaints) } + if got := ac.Kubelet.Configuration["maxPods"]; got != 42 { + t.Fatalf("Kubelet.Configuration.maxPods=%v, want 42", got) + } + if got := ac.Kubelet.Configuration["imageGCHighThresholdPercent"]; got != 90 { + t.Fatalf("Kubelet.Configuration.imageGCHighThresholdPercent=%v, want 90", got) + } + if got := ac.Kubelet.Configuration["imageGCLowThresholdPercent"]; got != 75 { + t.Fatalf("Kubelet.Configuration.imageGCLowThresholdPercent=%v, want 75", got) + } + logging, ok := ac.Kubelet.Configuration["logging"].(map[string]any) + if !ok { + t.Fatalf("Kubelet.Configuration.logging=%T, want map[string]any", ac.Kubelet.Configuration["logging"]) + } + if logging["verbosity"] != 4 { + t.Fatalf("Kubelet.Configuration.logging=%v, want verbosity=4", logging) + } + if ac.Kubelet.ImageCredentialProvider == nil { + t.Fatal("Kubelet.ImageCredentialProvider=nil, want provider") + } + if ac.Kubelet.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" { + t.Fatalf("Kubelet.ImageCredentialProvider.ConfigPath=%q", ac.Kubelet.ImageCredentialProvider.ConfigPath) + } + if ac.Kubelet.ImageCredentialProvider.BinDir != "/usr/local/lib/kubelet-credential-providers" { + t.Fatalf("Kubelet.ImageCredentialProvider.BinDir=%q", ac.Kubelet.ImageCredentialProvider.BinDir) + } } func TestToAgentConfig_NodeName(t *testing.T) { diff --git a/pkg/config/config.go b/pkg/config/config.go index ecc670c5..f1a9025a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -251,6 +251,17 @@ type KubeletConfig struct { ClusterFQDN string `json:"clusterFQDN,omitempty"` // Kubernetes API server FQDN from AKS RP bootstrap data CACertData string `json:"caCertData"` // Base64-encoded CA certificate data NodeIP string `json:"nodeIP"` // IP address to advertise as the node's primary IP (--node-ip kubelet flag) + + // ImageCredentialProvider configures kubelet's exec image credential + // provider. The referenced paths are inside the nspawn machine. + ImageCredentialProvider *ImageCredentialProviderConfig `json:"imageCredentialProvider,omitempty"` +} + +// ImageCredentialProviderConfig identifies the exec image credential provider +// configuration and binary directory inside the nspawn machine. +type ImageCredentialProviderConfig struct { + ConfigPath string `json:"configPath"` + BinDir string `json:"binDir"` } // NetworkingConfig is the AKS RP networking contract used by the agent at runtime. @@ -882,6 +893,9 @@ func (c *Config) validate() error { if err := c.Bootstrap.validate(); err != nil { return err } + if err := c.Node.Kubelet.validate(); err != nil { + return err + } if err := c.validateAuthSettings(); err != nil { return err @@ -893,6 +907,20 @@ func (c *Config) validate() error { return nil } +func (c *KubeletConfig) validate() error { + kubelet := agentconfig.AgentKubeletConfig{} + if c.ImageCredentialProvider != nil { + kubelet.ImageCredentialProvider = &agentconfig.ImageCredentialProvider{ + ConfigPath: c.ImageCredentialProvider.ConfigPath, + BinDir: c.ImageCredentialProvider.BinDir, + } + } + if err := kubelet.Validate(); err != nil { + return fmt.Errorf("invalid node.kubelet configuration: %w", err) + } + return nil +} + func (c *Config) validateAuthSettings() error { armAuthMethodCount := 0 for _, m := range []bool{c.IsARCEnabled(), c.IsSPConfigured(), c.IsMIConfigured()} { diff --git a/pkg/config/copy_test.go b/pkg/config/copy_test.go index cae05009..114b0a98 100644 --- a/pkg/config/copy_test.go +++ b/pkg/config/copy_test.go @@ -17,6 +17,12 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) { Node: NodeConfig{ Labels: map[string]string{"l": "1"}, Taints: []string{"dedicated=infra:NoSchedule"}, + Kubelet: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, + }, }, } @@ -47,6 +53,9 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) { if cfg.Components.Gantry == nil || copy.Components.Gantry == nil || cfg.Components.Gantry == copy.Components.Gantry { t.Fatalf("Gantry pointer shared or nil") } + if cfg.Node.Kubelet.ImageCredentialProvider == nil || copy.Node.Kubelet.ImageCredentialProvider == nil || cfg.Node.Kubelet.ImageCredentialProvider == copy.Node.Kubelet.ImageCredentialProvider { + t.Fatalf("ImageCredentialProvider pointer shared or nil") + } // Maps should not be shared (validate via independent mutation behavior). cfg.Azure.Arc.Tags["k"] = "orig" @@ -67,6 +76,11 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) { t.Fatalf("Node.Labels shared; orig=%q, want %q", cfg.Node.Labels["l"], "orig") } + copy.Node.Kubelet.ImageCredentialProvider.ConfigPath = "/etc/kubernetes/other.yaml" + if cfg.Node.Kubelet.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" { + t.Fatal("ImageCredentialProvider shared between config copies") + } + // Taints slice should not be shared. if len(copy.Node.Taints) != 1 || copy.Node.Taints[0] != "dedicated=infra:NoSchedule" { t.Fatalf("Node.Taints copy=%v, want [dedicated=infra:NoSchedule]", copy.Node.Taints) diff --git a/pkg/config/kubelet_config_test.go b/pkg/config/kubelet_config_test.go new file mode 100644 index 00000000..c2692118 --- /dev/null +++ b/pkg/config/kubelet_config_test.go @@ -0,0 +1,82 @@ +package config + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestKubeletConfigJSON(t *testing.T) { + t.Parallel() + + data := []byte(`{ + "imageCredentialProvider": { + "configPath": "/etc/kubernetes/credential-provider.yaml", + "binDir": "/usr/local/lib/kubelet-credential-providers" + } + }`) + var config KubeletConfig + if err := json.Unmarshal(data, &config); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if config.ImageCredentialProvider == nil { + t.Fatal("ImageCredentialProvider=nil, want provider") + } + if config.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" || config.ImageCredentialProvider.BinDir != "/usr/local/lib/kubelet-credential-providers" { + t.Fatalf("ImageCredentialProvider=%#v", config.ImageCredentialProvider) + } +} + +func TestKubeletConfigValidate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + config KubeletConfig + wantErr string + }{ + {name: "empty"}, + { + name: "image credential provider", + config: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, + }, + }, + { + name: "missing provider config path", + config: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{BinDir: "/usr/bin"}, + }, + wantErr: "ConfigPath", + }, + { + name: "relative provider binary directory", + config: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "bin", + }, + }, + wantErr: "absolute path", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := tt.config.validate() + if tt.wantErr == "" { + if err != nil { + t.Fatalf("validate: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("validate error=%v, want substring %q", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/daemon/nodeoperator.go b/pkg/daemon/nodeoperator.go index f255cb9b..1a90a605 100644 --- a/pkg/daemon/nodeoperator.go +++ b/pkg/daemon/nodeoperator.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log/slog" + "maps" + "slices" "github.com/Azure/AKSFlexNode/pkg/aksmachine" "github.com/Azure/AKSFlexNode/pkg/config" @@ -84,9 +86,7 @@ func (o *nspawnNodeOperator) ApplyGoalState(ctx context.Context, log *slog.Logge // TODO: This per-goal config copy/mutation is not ideal. Refactor goal-state // resolution to avoid rewriting shared config-shaped data here. cfg := o.cfg.DeepCopy() - if goal.KubernetesVersion != "" { - cfg.Components.Kubernetes = goal.KubernetesVersion - } + applyMachineGoalToConfig(cfg, goal) oldMachine := active.Name newMachine := goalstates.AlternateMachine(oldMachine) log.Info("starting nspawn machine goal-state apply", @@ -114,6 +114,30 @@ func (o *nspawnNodeOperator) ApplyGoalState(ctx context.Context, log *slog.Logge return newState, nil } +func applyMachineGoalToConfig(cfg *config.Config, goal aksmachine.GoalState) { + if goal.KubernetesVersion != "" { + cfg.Components.Kubernetes = goal.KubernetesVersion + } + // Zero values represent fields omitted by the Machine API. Preserve local + // defaults in that case; non-nil empty labels or taints still explicitly + // clear those collections. + if goal.MaxPods != 0 { + cfg.Node.MaxPods = goal.MaxPods + } + if goal.NodeLabels != nil { + cfg.Node.Labels = maps.Clone(goal.NodeLabels) + } + if goal.NodeTaints != nil { + cfg.Node.Taints = slices.Clone(goal.NodeTaints) + } + if goal.KubeletConfig.ImageGCHighThreshold != 0 { + cfg.Node.Kubelet.ImageGCHighThreshold = goal.KubeletConfig.ImageGCHighThreshold + } + if goal.KubeletConfig.ImageGCLowThreshold != 0 { + cfg.Node.Kubelet.ImageGCLowThreshold = goal.KubeletConfig.ImageGCLowThreshold + } +} + func (o *nspawnNodeOperator) ResetNode(ctx context.Context, log *slog.Logger) error { return phases.ExecuteTask(ctx, log, ResetNode(log)) } diff --git a/pkg/daemon/nodeoperator_test.go b/pkg/daemon/nodeoperator_test.go index ae08a2cb..56201b2a 100644 --- a/pkg/daemon/nodeoperator_test.go +++ b/pkg/daemon/nodeoperator_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" + "github.com/Azure/AKSFlexNode/pkg/aksmachine" + "github.com/Azure/AKSFlexNode/pkg/config" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -57,6 +59,91 @@ func TestFindActiveMachine(t *testing.T) { } } +func TestApplyMachineGoalToConfig(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Components: config.ComponentsConfig{Kubernetes: "1.33.0"}, + Node: config.NodeConfig{ + MaxPods: 110, + Labels: map[string]string{"source": "local"}, + Taints: []string{"local=true:NoSchedule"}, + Kubelet: config.KubeletConfig{ + Verbosity: 4, + ImageGCHighThreshold: 85, + ImageGCLowThreshold: 80, + ImageCredentialProvider: &config.ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, + }, + }, + } + goal := aksmachine.GoalState{ + KubernetesVersion: "1.34.1", + MaxPods: 42, + NodeLabels: map[string]string{"source": "remote"}, + NodeTaints: []string{"remote=true:NoExecute"}, + KubeletConfig: aksmachine.KubeletConfig{ + ImageGCHighThreshold: 90, + ImageGCLowThreshold: 75, + }, + } + + applyMachineGoalToConfig(cfg, goal) + + if cfg.Components.Kubernetes != "1.34.1" || cfg.Node.MaxPods != 42 { + t.Fatalf("version=%q maxPods=%d, want 1.34.1 and 42", cfg.Components.Kubernetes, cfg.Node.MaxPods) + } + if cfg.Node.Labels["source"] != "remote" || cfg.Node.Taints[0] != "remote=true:NoExecute" { + t.Fatalf("labels=%v taints=%v, want remote settings", cfg.Node.Labels, cfg.Node.Taints) + } + if cfg.Node.Kubelet.ImageGCHighThreshold != 90 || cfg.Node.Kubelet.ImageGCLowThreshold != 75 { + t.Fatalf("kubelet GC thresholds=%d/%d, want 90/75", cfg.Node.Kubelet.ImageGCHighThreshold, cfg.Node.Kubelet.ImageGCLowThreshold) + } + if cfg.Node.Kubelet.Verbosity != 4 { + t.Fatal("local-only kubelet verbosity was not preserved") + } + if cfg.Node.Kubelet.ImageCredentialProvider == nil { + t.Fatal("local image credential provider was not preserved") + } + + goal.NodeLabels["source"] = "mutated" + goal.NodeTaints[0] = "mutated=true:NoSchedule" + if cfg.Node.Labels["source"] != "remote" || cfg.Node.Taints[0] != "remote=true:NoExecute" { + t.Fatal("applied config shares mutable goal state") + } +} + +func TestApplyMachineGoalToConfigPreservesSettingsOmittedByMachineAPI(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Components: config.ComponentsConfig{Kubernetes: "1.33.0"}, + Node: config.NodeConfig{ + MaxPods: 110, + Labels: map[string]string{"source": "local"}, + Taints: []string{"local=true:NoSchedule"}, + Kubelet: config.KubeletConfig{ + ImageGCHighThreshold: 85, + ImageGCLowThreshold: 80, + }, + }, + } + + applyMachineGoalToConfig(cfg, aksmachine.GoalState{KubernetesVersion: "1.34.1"}) + + if cfg.Components.Kubernetes != "1.34.1" { + t.Fatalf("Kubernetes version=%q, want 1.34.1", cfg.Components.Kubernetes) + } + if cfg.Node.MaxPods != 110 || cfg.Node.Labels["source"] != "local" || cfg.Node.Taints[0] != "local=true:NoSchedule" { + t.Fatalf("omitted machine settings replaced local values: %#v", cfg.Node) + } + if cfg.Node.Kubelet.ImageGCHighThreshold != 85 || cfg.Node.Kubelet.ImageGCLowThreshold != 80 { + t.Fatalf("omitted GC thresholds replaced local values: %#v", cfg.Node.Kubelet) + } +} + type testStateStore struct { state *State } From b990c58cf12c1c80d4a42d41300c615188c84989 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 4 Aug 2026 23:20:14 +0000 Subject: [PATCH 11/16] refactor: defer machine goal persistence --- pkg/daemon/nodeoperator.go | 30 ++---------- pkg/daemon/nodeoperator_test.go | 87 --------------------------------- 2 files changed, 3 insertions(+), 114 deletions(-) diff --git a/pkg/daemon/nodeoperator.go b/pkg/daemon/nodeoperator.go index 1a90a605..f255cb9b 100644 --- a/pkg/daemon/nodeoperator.go +++ b/pkg/daemon/nodeoperator.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "log/slog" - "maps" - "slices" "github.com/Azure/AKSFlexNode/pkg/aksmachine" "github.com/Azure/AKSFlexNode/pkg/config" @@ -86,7 +84,9 @@ func (o *nspawnNodeOperator) ApplyGoalState(ctx context.Context, log *slog.Logge // TODO: This per-goal config copy/mutation is not ideal. Refactor goal-state // resolution to avoid rewriting shared config-shaped data here. cfg := o.cfg.DeepCopy() - applyMachineGoalToConfig(cfg, goal) + if goal.KubernetesVersion != "" { + cfg.Components.Kubernetes = goal.KubernetesVersion + } oldMachine := active.Name newMachine := goalstates.AlternateMachine(oldMachine) log.Info("starting nspawn machine goal-state apply", @@ -114,30 +114,6 @@ func (o *nspawnNodeOperator) ApplyGoalState(ctx context.Context, log *slog.Logge return newState, nil } -func applyMachineGoalToConfig(cfg *config.Config, goal aksmachine.GoalState) { - if goal.KubernetesVersion != "" { - cfg.Components.Kubernetes = goal.KubernetesVersion - } - // Zero values represent fields omitted by the Machine API. Preserve local - // defaults in that case; non-nil empty labels or taints still explicitly - // clear those collections. - if goal.MaxPods != 0 { - cfg.Node.MaxPods = goal.MaxPods - } - if goal.NodeLabels != nil { - cfg.Node.Labels = maps.Clone(goal.NodeLabels) - } - if goal.NodeTaints != nil { - cfg.Node.Taints = slices.Clone(goal.NodeTaints) - } - if goal.KubeletConfig.ImageGCHighThreshold != 0 { - cfg.Node.Kubelet.ImageGCHighThreshold = goal.KubeletConfig.ImageGCHighThreshold - } - if goal.KubeletConfig.ImageGCLowThreshold != 0 { - cfg.Node.Kubelet.ImageGCLowThreshold = goal.KubeletConfig.ImageGCLowThreshold - } -} - func (o *nspawnNodeOperator) ResetNode(ctx context.Context, log *slog.Logger) error { return phases.ExecuteTask(ctx, log, ResetNode(log)) } diff --git a/pkg/daemon/nodeoperator_test.go b/pkg/daemon/nodeoperator_test.go index 56201b2a..ae08a2cb 100644 --- a/pkg/daemon/nodeoperator_test.go +++ b/pkg/daemon/nodeoperator_test.go @@ -4,8 +4,6 @@ import ( "context" "testing" - "github.com/Azure/AKSFlexNode/pkg/aksmachine" - "github.com/Azure/AKSFlexNode/pkg/config" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -59,91 +57,6 @@ func TestFindActiveMachine(t *testing.T) { } } -func TestApplyMachineGoalToConfig(t *testing.T) { - t.Parallel() - - cfg := &config.Config{ - Components: config.ComponentsConfig{Kubernetes: "1.33.0"}, - Node: config.NodeConfig{ - MaxPods: 110, - Labels: map[string]string{"source": "local"}, - Taints: []string{"local=true:NoSchedule"}, - Kubelet: config.KubeletConfig{ - Verbosity: 4, - ImageGCHighThreshold: 85, - ImageGCLowThreshold: 80, - ImageCredentialProvider: &config.ImageCredentialProviderConfig{ - ConfigPath: "/etc/kubernetes/credential-provider.yaml", - BinDir: "/usr/local/lib/kubelet-credential-providers", - }, - }, - }, - } - goal := aksmachine.GoalState{ - KubernetesVersion: "1.34.1", - MaxPods: 42, - NodeLabels: map[string]string{"source": "remote"}, - NodeTaints: []string{"remote=true:NoExecute"}, - KubeletConfig: aksmachine.KubeletConfig{ - ImageGCHighThreshold: 90, - ImageGCLowThreshold: 75, - }, - } - - applyMachineGoalToConfig(cfg, goal) - - if cfg.Components.Kubernetes != "1.34.1" || cfg.Node.MaxPods != 42 { - t.Fatalf("version=%q maxPods=%d, want 1.34.1 and 42", cfg.Components.Kubernetes, cfg.Node.MaxPods) - } - if cfg.Node.Labels["source"] != "remote" || cfg.Node.Taints[0] != "remote=true:NoExecute" { - t.Fatalf("labels=%v taints=%v, want remote settings", cfg.Node.Labels, cfg.Node.Taints) - } - if cfg.Node.Kubelet.ImageGCHighThreshold != 90 || cfg.Node.Kubelet.ImageGCLowThreshold != 75 { - t.Fatalf("kubelet GC thresholds=%d/%d, want 90/75", cfg.Node.Kubelet.ImageGCHighThreshold, cfg.Node.Kubelet.ImageGCLowThreshold) - } - if cfg.Node.Kubelet.Verbosity != 4 { - t.Fatal("local-only kubelet verbosity was not preserved") - } - if cfg.Node.Kubelet.ImageCredentialProvider == nil { - t.Fatal("local image credential provider was not preserved") - } - - goal.NodeLabels["source"] = "mutated" - goal.NodeTaints[0] = "mutated=true:NoSchedule" - if cfg.Node.Labels["source"] != "remote" || cfg.Node.Taints[0] != "remote=true:NoExecute" { - t.Fatal("applied config shares mutable goal state") - } -} - -func TestApplyMachineGoalToConfigPreservesSettingsOmittedByMachineAPI(t *testing.T) { - t.Parallel() - - cfg := &config.Config{ - Components: config.ComponentsConfig{Kubernetes: "1.33.0"}, - Node: config.NodeConfig{ - MaxPods: 110, - Labels: map[string]string{"source": "local"}, - Taints: []string{"local=true:NoSchedule"}, - Kubelet: config.KubeletConfig{ - ImageGCHighThreshold: 85, - ImageGCLowThreshold: 80, - }, - }, - } - - applyMachineGoalToConfig(cfg, aksmachine.GoalState{KubernetesVersion: "1.34.1"}) - - if cfg.Components.Kubernetes != "1.34.1" { - t.Fatalf("Kubernetes version=%q, want 1.34.1", cfg.Components.Kubernetes) - } - if cfg.Node.MaxPods != 110 || cfg.Node.Labels["source"] != "local" || cfg.Node.Taints[0] != "local=true:NoSchedule" { - t.Fatalf("omitted machine settings replaced local values: %#v", cfg.Node) - } - if cfg.Node.Kubelet.ImageGCHighThreshold != 85 || cfg.Node.Kubelet.ImageGCLowThreshold != 80 { - t.Fatalf("omitted GC thresholds replaced local values: %#v", cfg.Node.Kubelet) - } -} - type testStateStore struct { state *State } From 584b155e4960318b668fd21166b089658d7bc17c Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 4 Aug 2026 23:31:53 +0000 Subject: [PATCH 12/16] chore: bump unbounded to v0.2.3-alpha.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 890ee6e9..a619597a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260804230755-37f986116380 + github.com/Azure/unbounded v0.2.3-alpha.0 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 1877f777..421b1fc1 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260804230755-37f986116380 h1:MtfQZ33ZG2T8Vu/8NsjKAswWoBFZi/cg5wwoCu2je8Q= -github.com/Azure/unbounded v0.2.3-0.20260804230755-37f986116380/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-alpha.0 h1:Bi1NhKUACOUDXxUIEx3JKJL8dQ1TKsRBUlIahAm1cHc= +github.com/Azure/unbounded v0.2.3-alpha.0/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 723fc4bd7f67a09efd70f1bf6eaa68831a791ec9 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 5 Aug 2026 01:48:27 +0000 Subject: [PATCH 13/16] test: validate generated cluster DNS config after repave --- hack/e2e/lib/upgrade-drift.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/e2e/lib/upgrade-drift.sh b/hack/e2e/lib/upgrade-drift.sh index c2c3970a..bf6f6ffe 100644 --- a/hack/e2e/lib/upgrade-drift.sh +++ b/hack/e2e/lib/upgrade-drift.sh @@ -251,7 +251,8 @@ test -n "${machine}" ! ip link show localdns >/dev/null 2>&1 ! iptables -w -t raw -S | grep -q 'unbounded-localdns: skip conntrack' systemd-run --quiet --pipe --wait --machine="${machine}" \ - grep -q -- "--cluster-dns=${CLUSTER_DNS}" /etc/systemd/system/kubelet.service.d/20-node-config.conf + grep -A1 -F 'clusterDNS:' /var/lib/kubelet/config.yaml \ + | grep -q -F -- "- ${CLUSTER_DNS}" REMOTE log_success "MSI LocalDNS disable-through-repave validation passed" From ba0805050dbd122db7a53a840a2e58f4e39b9ae9 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 5 Aug 2026 19:26:39 +0000 Subject: [PATCH 14/16] chore: bump unbounded to v0.2.3-alpha.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a619597a..346bf486 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-alpha.0 + github.com/Azure/unbounded v0.2.3-alpha.1 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 421b1fc1..f92298b5 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-alpha.0 h1:Bi1NhKUACOUDXxUIEx3JKJL8dQ1TKsRBUlIahAm1cHc= -github.com/Azure/unbounded v0.2.3-alpha.0/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-alpha.1 h1:nHl50aQGQhWGgit4fCa3E4xrlgxDQaACJhTgKhCW2/A= +github.com/Azure/unbounded v0.2.3-alpha.1/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 3b41fdfdcfe9faee794b22dfacee1c2598a64491 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 6 Aug 2026 01:00:07 +0000 Subject: [PATCH 15/16] chore: bump unbounded alpha branch --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 346bf486..211d97f6 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-alpha.1 + github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806005817-9a1fdb16f5d3 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index f92298b5..dc7f7f15 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-alpha.1 h1:nHl50aQGQhWGgit4fCa3E4xrlgxDQaACJhTgKhCW2/A= -github.com/Azure/unbounded v0.2.3-alpha.1/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806005817-9a1fdb16f5d3 h1:Vq39k7INl941nhCoEZA/t3VYFghTtXU4/lfXDPhISd8= +github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806005817-9a1fdb16f5d3/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From ba120bdde516bfe2992876f094b5b40a403c84bd Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 6 Aug 2026 01:04:12 +0000 Subject: [PATCH 16/16] chore: bump unbounded for LocalDNS node IP discovery --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 211d97f6..a69c755e 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806005817-9a1fdb16f5d3 + github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806010226-43ec4d137b23 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index dc7f7f15..cb88b8ae 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806005817-9a1fdb16f5d3 h1:Vq39k7INl941nhCoEZA/t3VYFghTtXU4/lfXDPhISd8= -github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806005817-9a1fdb16f5d3/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806010226-43ec4d137b23 h1:dhcPkHz7irmPQ8GU7IDhgZr8ag3Y1j3EeYS6znV4yug= +github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806010226-43ec4d137b23/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=