From e54eeac429eeaa0af8dcc5b69634c9bd6842eaca Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 29 Jul 2026 04:52:04 +0000 Subject: [PATCH 01/17] feat: support AKS LocalDNS profiles --- docs/usages/configuration.md | 44 +++++++ hack/e2e/lib/node-join-msi.sh | 32 ++++- hack/e2e/lib/validate.sh | 26 ++++ pkg/config/adapter.go | 19 ++- pkg/config/config.go | 8 +- pkg/config/localdns.go | 191 ++++++++++++++++++++++++++++ pkg/config/localdns_adapter_test.go | 50 ++++++++ pkg/config/localdns_test.go | 64 ++++++++++ 8 files changed, 430 insertions(+), 4 deletions(-) create mode 100644 pkg/config/localdns.go create mode 100644 pkg/config/localdns_adapter_test.go create mode 100644 pkg/config/localdns_test.go diff --git a/docs/usages/configuration.md b/docs/usages/configuration.md index 11d8084e..87bce17a 100644 --- a/docs/usages/configuration.md +++ b/docs/usages/configuration.md @@ -122,6 +122,50 @@ At least one join or Azure authentication method must be configured. `azure.boot |------|------|-------------|--------------| | `networking.dnsServiceIP` | string | Cluster DNS service IP. | `10.0.0.10` | | `networking.cniVersion` | string | Optional CNI plugin version override. | `v1.6.2` | +| `networking.localDNS` | object | Optional AKS LocalDNS profile using the same `mode`, `vnetDNSOverrides`, and `kubeDNSOverrides` shape accepted by `az aks nodepool --localdns-config`. | `{ "mode": "Required" }` | + +### AKS LocalDNS + +AKS Flex Node accepts the official AKS LocalDNS JSON profile under +`networking.localDNS`. See [Configure LocalDNS in +AKS](https://learn.microsoft.com/azure/aks/localdns-custom) for the supported +fields and values. `Required` enables LocalDNS, `Disabled` disables it through +repave, and `Preferred` validates the profile without enabling the service. + +```json +{ + "networking": { + "dnsServiceIP": "10.0.0.10", + "localDNS": { + "mode": "Required", + "vnetDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "ForceTCP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } +} +``` ## Node diff --git a/hack/e2e/lib/node-join-msi.sh b/hack/e2e/lib/node-join-msi.sh index e9167574..053d3593 100644 --- a/hack/e2e/lib/node-join-msi.sh +++ b/hack/e2e/lib/node-join-msi.sh @@ -55,7 +55,8 @@ node_join_msi() { "node": { "kubelet": { "clusterFQDN": "${server_url}", - "caCertData": "${ca_cert_data}" + "caCertData": "${ca_cert_data}", + "nodeIP": "${vm_ip}" } }, "agent": { @@ -67,6 +68,35 @@ node_join_msi() { }, "requireMachineRegistration": true }, + "networking": { + "localDNS": { + "mode": "Required", + "vnetDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "ForceTCP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + }, "components": { "kubernetes": "${E2E_KUBERNETES_VERSION}", "containerd": "${E2E_CONTAINERD_VERSION}", diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 02ab9f6c..44ddd7a3 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -168,6 +168,31 @@ REMOTE } # --------------------------------------------------------------------------- +# validate_localdns_status - Verify nspawn LocalDNS on the selected VM. +validate_localdns_status() { + local vm_ip="$1" + log_info "Validating LocalDNS on ${vm_ip}..." + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail +machine=$(sudo machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}') +test -n "${machine}" +sudo systemd-run --quiet --pipe --wait --machine="${machine}" systemctl is-active --quiet localdns.service +sudo systemd-run --quiet --pipe --wait --machine="${machine}" grep -qx 'nameserver 169.254.10.10' /etc/resolv.conf +sudo ip address show dev localdns | grep -q '169.254.10.10/32' +sudo ip address show dev localdns | grep -q '169.254.10.11/32' +for chain in OUTPUT PREROUTING; do + for address in 169.254.10.10 169.254.10.11; do + for protocol in tcp udp; do + sudo iptables -w -t raw -C "${chain}" -m comment \ + --comment 'unbounded-localdns: skip conntrack' \ + -p "${protocol}" -d "${address}" --dport 53 -j NOTRACK + done + done +done +REMOTE + log_success "LocalDNS validation passed on ${vm_ip}" +} + # validate_all_nodes - Check all MSI, token, offline, and kubeadm VMs joined # --------------------------------------------------------------------------- validate_all_nodes() { @@ -206,6 +231,7 @@ validate_all_nodes() { validate_node_ip "${token_vm_name}" "${token_vm_private_ip}" || failed=1 validate_node_ip "${offline_vm_name}" "${offline_vm_private_ip}" || failed=1 validate_npd_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 + validate_localdns_status "${msi_vm_ip}" || failed=1 validate_npd_status "${token_vm_name}" "${token_vm_ip}" || failed=1 # TODO: re-enable once NPD is included in the upstream Unbounded bootstrap # artifact bundle and resolver used by offline artifact mode. diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index dfd302b3..f9e16a30 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "maps" agentconfig "github.com/Azure/unbounded/pkg/agent/config" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -26,6 +27,11 @@ const ( // // cfg.Node.Kubelet.ClusterFQDN and cfg.Node.Kubelet.CACertData must be populated. func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { + labels := maps.Clone(cfg.Node.Labels) + if labels == nil { + labels = map[string]string{} + } + ac := &agentconfig.AgentConfig{ MachineName: machineName, NodeName: cfg.Agent.NodeName, @@ -40,7 +46,7 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { Kubelet: agentconfig.AgentKubeletConfig{ ApiServer: cfg.APIServerURL(), NodeIP: cfg.Node.Kubelet.NodeIP, - Labels: cfg.Node.Labels, + Labels: labels, RegisterWithTaints: cfg.Node.Taints, }, CRI: agentconfig.CRIConfig{ @@ -57,6 +63,17 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { }, } + if profile := cfg.Networking.LocalDNS; profile != nil { + corefile, _ := profile.CorefileTemplate() // Config validation runs before adaptation. + ac.LocalDNS = &agentconfig.AgentLocalDNSConfig{ + Enabled: profile.Enabled(), + CorefileTemplate: corefile, + } + if profile.Enabled() { + labels["kubernetes.azure.com/localdns"] = "enabled" + } + } + if cfg.Bootstrap.OfflineArtifacts.Source != "" { ac.OfflineArtifacts = &agentconfig.AgentOfflineArtifacts{ Source: cfg.Bootstrap.OfflineArtifacts.Source, diff --git a/pkg/config/config.go b/pkg/config/config.go index ecc670c5..98bdb546 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -255,8 +255,9 @@ type KubeletConfig struct { // NetworkingConfig is the AKS RP networking contract used by the agent at runtime. type NetworkingConfig struct { - DNSServiceIP string `json:"dnsServiceIP,omitempty"` // Cluster DNS service IP (default: 10.0.0.10 for AKS) - CNIVersion string `json:"cniVersion,omitempty"` + DNSServiceIP string `json:"dnsServiceIP,omitempty"` // Cluster DNS service IP (default: 10.0.0.10 for AKS) + CNIVersion string `json:"cniVersion,omitempty"` + LocalDNS *LocalDNSProfile `json:"localDNS,omitempty"` } // NPDConfig holds configuration settings for the Node Problem Detector (NPD). @@ -882,6 +883,9 @@ func (c *Config) validate() error { if err := c.Bootstrap.validate(); err != nil { return err } + if err := c.Networking.LocalDNS.validate(); err != nil { + return fmt.Errorf("invalid networking.localDNS: %w", err) + } if err := c.validateAuthSettings(); err != nil { return err diff --git a/pkg/config/localdns.go b/pkg/config/localdns.go new file mode 100644 index 00000000..d552c806 --- /dev/null +++ b/pkg/config/localdns.go @@ -0,0 +1,191 @@ +package config + +import ( + "errors" + "fmt" + "sort" + "strings" +) + +const ( + LocalDNSModeRequired = "Required" + LocalDNSModePreferred = "Preferred" + LocalDNSModeDisabled = "Disabled" +) + +// LocalDNSProfile mirrors the AKS node-pool LocalDNS configuration contract. +type LocalDNSProfile struct { + Mode string `json:"mode"` + VnetDNSOverrides map[string]LocalDNSOverride `json:"vnetDNSOverrides,omitempty"` + KubeDNSOverrides map[string]LocalDNSOverride `json:"kubeDNSOverrides,omitempty"` +} + +// LocalDNSOverride configures one CoreDNS server block. +type LocalDNSOverride struct { + QueryLogging string `json:"queryLogging,omitempty"` + Protocol string `json:"protocol,omitempty"` + ForwardDestination string `json:"forwardDestination,omitempty"` + ForwardPolicy string `json:"forwardPolicy,omitempty"` + MaxConcurrent int `json:"maxConcurrent,omitempty"` + CacheDurationInSeconds int `json:"cacheDurationInSeconds,omitempty"` + ServeStaleDurationInSeconds int `json:"serveStaleDurationInSeconds,omitempty"` + ServeStale string `json:"serveStale,omitempty"` +} + +func (p *LocalDNSProfile) validate() error { + if p == nil { + return nil + } + if p.Mode != LocalDNSModeRequired && p.Mode != LocalDNSModePreferred && p.Mode != LocalDNSModeDisabled { + return fmt.Errorf("mode must be Required, Preferred, or Disabled") + } + var errs []error + for class, overrides := range map[string]map[string]LocalDNSOverride{ + "vnetDNSOverrides": p.VnetDNSOverrides, + "kubeDNSOverrides": p.KubeDNSOverrides, + } { + for zone, override := range overrides { + if strings.TrimSpace(zone) == "" || strings.ContainsAny(zone, "{} \t\r\n") { + errs = append(errs, fmt.Errorf("%s zone %q is invalid", class, zone)) + } + if err := override.validate(); err != nil { + errs = append(errs, fmt.Errorf("%s[%q]: %w", class, zone, err)) + } + } + } + return errors.Join(errs...) +} + +func (o LocalDNSOverride) validate() error { + var errs []error + if o.QueryLogging != "" && o.QueryLogging != "Error" && o.QueryLogging != "Log" { + errs = append(errs, fmt.Errorf("queryLogging must be Error or Log")) + } + if o.Protocol != "" && o.Protocol != "PreferUDP" && o.Protocol != "ForceTCP" { + errs = append(errs, fmt.Errorf("protocol must be PreferUDP or ForceTCP")) + } + if o.ForwardDestination != "" && o.ForwardDestination != "VnetDNS" && o.ForwardDestination != "ClusterCoreDNS" { + errs = append(errs, fmt.Errorf("forwardDestination must be VnetDNS or ClusterCoreDNS")) + } + if o.ForwardPolicy != "" && o.ForwardPolicy != "Sequential" && o.ForwardPolicy != "RoundRobin" && o.ForwardPolicy != "Random" { + errs = append(errs, fmt.Errorf("forwardPolicy must be Sequential, RoundRobin, or Random")) + } + if o.ServeStale != "" && o.ServeStale != "Disable" && o.ServeStale != "Verify" && o.ServeStale != "Immediate" { + errs = append(errs, fmt.Errorf("serveStale must be Disable, Verify, or Immediate")) + } + for name, value := range map[string]int{ + "maxConcurrent": o.MaxConcurrent, + "cacheDurationInSeconds": o.CacheDurationInSeconds, + "serveStaleDurationInSeconds": o.ServeStaleDurationInSeconds, + } { + if value < 0 { + errs = append(errs, fmt.Errorf("%s must not be negative", name)) + } + } + return errors.Join(errs...) +} + +// Enabled reports whether the profile requires LocalDNS installation. +func (p *LocalDNSProfile) Enabled() bool { + return p != nil && p.Mode == LocalDNSModeRequired +} + +// CorefileTemplate renders AKS LocalDNS policy into an Unbounded Corefile template. +func (p *LocalDNSProfile) CorefileTemplate() (string, error) { + if p == nil { + return "", nil + } + if err := p.validate(); err != nil { + return "", err + } + if !p.Enabled() { + return "", nil + } + + vnet := p.VnetDNSOverrides + if len(vnet) == 0 { + vnet = map[string]LocalDNSOverride{".": {ForwardDestination: "VnetDNS"}} + } + kube := p.KubeDNSOverrides + if len(kube) == 0 { + kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} + } + + var out strings.Builder + out.WriteString("health-check.localdns.local:53 {\n bind {{ .NodeListenerIP }} {{ .ClusterListenerIP }}\n whoami\n}\n\n") + renderLocalDNSBlocks(&out, vnet, "{{ .NodeListenerIP }}", "VnetDNS") + renderLocalDNSBlocks(&out, kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS") + return out.String(), nil +} + +func renderLocalDNSBlocks(out *strings.Builder, overrides map[string]LocalDNSOverride, listener, defaultDestination string) { + zones := make([]string, 0, len(overrides)) + for zone := range overrides { + zones = append(zones, zone) + } + sort.Strings(zones) + for _, zone := range zones { + o := withLocalDNSDefaults(overrides[zone], defaultDestination) + fmt.Fprintf(out, "%s:53 {\n", zone) + if o.QueryLogging == "Log" { + out.WriteString(" log\n") + } else { + out.WriteString(" errors\n") + } + fmt.Fprintf(out, " bind %s\n", listener) + upstream := "{{ .NodeUpstreamIPsJoined }}" + if o.ForwardDestination == "ClusterCoreDNS" { + upstream = "{{ .ClusterDNSServiceIP }}" + } + fmt.Fprintf(out, " forward . %s {\n", upstream) + if o.Protocol == "ForceTCP" { + out.WriteString(" force_tcp\n") + } + fmt.Fprintf(out, " policy %s\n max_concurrent %d\n }\n", localDNSForwardPolicy(o.ForwardPolicy), o.MaxConcurrent) + fmt.Fprintf(out, " ready %s:8181\n", listener) + fmt.Fprintf(out, " cache %d {\n success 9984\n denial 9984\n", o.CacheDurationInSeconds) + if o.ServeStale != "Disable" { + fmt.Fprintf(out, " serve_stale %ds %s\n", o.ServeStaleDurationInSeconds, strings.ToLower(o.ServeStale)) + } + out.WriteString(" servfail 0\n }\n loop\n prometheus {{ .MetricsAddress }}\n}\n\n") + } +} + +func withLocalDNSDefaults(o LocalDNSOverride, destination string) LocalDNSOverride { + if o.QueryLogging == "" { + o.QueryLogging = "Error" + } + if o.Protocol == "" { + o.Protocol = "ForceTCP" + } + if o.ForwardDestination == "" { + o.ForwardDestination = destination + } + if o.ForwardPolicy == "" { + o.ForwardPolicy = "Sequential" + } + if o.MaxConcurrent == 0 { + o.MaxConcurrent = 1000 + } + if o.CacheDurationInSeconds == 0 { + o.CacheDurationInSeconds = 3600 + } + if o.ServeStaleDurationInSeconds == 0 { + o.ServeStaleDurationInSeconds = 3600 + } + if o.ServeStale == "" { + o.ServeStale = "Immediate" + } + return o +} + +func localDNSForwardPolicy(policy string) string { + switch policy { + case "RoundRobin": + return "round_robin" + case "Random": + return "random" + default: + return "sequential" + } +} diff --git a/pkg/config/localdns_adapter_test.go b/pkg/config/localdns_adapter_test.go new file mode 100644 index 00000000..eba36356 --- /dev/null +++ b/pkg/config/localdns_adapter_test.go @@ -0,0 +1,50 @@ +package config + +import ( + "strings" + "testing" +) + +func TestToAgentConfigLocalDNS(t *testing.T) { + t.Parallel() + + cfg := &Config{ + Networking: NetworkingConfig{ + DNSServiceIP: "10.0.0.10", + LocalDNS: &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + KubeDNSOverrides: map[string]LocalDNSOverride{ + ".": {ForwardDestination: "ClusterCoreDNS"}, + }, + }, + }, + Node: NodeConfig{Labels: map[string]string{"example": "value"}}, + } + + got := ToAgentConfig(cfg, "kube1") + if got.LocalDNS == nil || !got.LocalDNS.Enabled { + t.Fatalf("LocalDNS = %#v, want enabled", got.LocalDNS) + } + if !strings.Contains(got.LocalDNS.CorefileTemplate, "{{ .ClusterDNSServiceIP }}") { + t.Fatalf("CorefileTemplate missing cluster DNS placeholder:\n%s", got.LocalDNS.CorefileTemplate) + } + if got.Kubelet.Labels["kubernetes.azure.com/localdns"] != "enabled" { + t.Fatalf("LocalDNS node label missing: %#v", got.Kubelet.Labels) + } + if got.Cluster.ClusterDNS != "10.0.0.10" { + t.Fatalf("ClusterDNS = %q, want original service IP", got.Cluster.ClusterDNS) + } +} + +func TestToAgentConfigDisabledLocalDNS(t *testing.T) { + t.Parallel() + + cfg := &Config{Networking: NetworkingConfig{ + DNSServiceIP: "10.0.0.10", + LocalDNS: &LocalDNSProfile{Mode: LocalDNSModeDisabled}, + }} + got := ToAgentConfig(cfg, "kube1") + if got.LocalDNS == nil || got.LocalDNS.Enabled { + t.Fatalf("LocalDNS = %#v, want explicitly disabled", got.LocalDNS) + } +} diff --git a/pkg/config/localdns_test.go b/pkg/config/localdns_test.go new file mode 100644 index 00000000..04e92abe --- /dev/null +++ b/pkg/config/localdns_test.go @@ -0,0 +1,64 @@ +package config + +import ( + "strings" + "testing" +) + +func TestLocalDNSProfileCorefileTemplate(t *testing.T) { + t.Parallel() + + profile := &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + VnetDNSOverrides: map[string]LocalDNSOverride{ + ".": {ForwardDestination: "VnetDNS", Protocol: "PreferUDP"}, + }, + KubeDNSOverrides: map[string]LocalDNSOverride{ + ".": {ForwardDestination: "ClusterCoreDNS", Protocol: "ForceTCP", ForwardPolicy: "RoundRobin"}, + }, + } + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatalf("CorefileTemplate() error = %v", err) + } + for _, want := range []string{ + "bind {{ .NodeListenerIP }}", + "forward . {{ .NodeUpstreamIPsJoined }}", + "bind {{ .ClusterListenerIP }}", + "forward . {{ .ClusterDNSServiceIP }}", + "force_tcp", + "policy round_robin", + } { + if !strings.Contains(got, want) { + t.Fatalf("CorefileTemplate() missing %q:\n%s", want, got) + } + } +} + +func TestLocalDNSProfileValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + profile *LocalDNSProfile + wantErr string + }{ + {name: "required", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired}}, + {name: "preferred", profile: &LocalDNSProfile{Mode: LocalDNSModePreferred}}, + {name: "disabled", profile: &LocalDNSProfile{Mode: LocalDNSModeDisabled}}, + {name: "invalid mode", profile: &LocalDNSProfile{Mode: "On"}, wantErr: "mode"}, + {name: "invalid protocol", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {Protocol: "TLS"}}}, wantErr: "protocol"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + err := test.profile.validate() + if test.wantErr == "" && err != nil { + t.Fatalf("validate() error = %v", err) + } + if test.wantErr != "" && (err == nil || !strings.Contains(err.Error(), test.wantErr)) { + t.Fatalf("validate() error = %v, want %q", err, test.wantErr) + } + }) + } +} From f918eefddc9476b6b8dba0cf25c6d526298c590e Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 29 Jul 2026 05:08:51 +0000 Subject: [PATCH 02/17] refactor: render LocalDNS blocks with templates --- pkg/config/localdns.go | 105 +++++++++++++++++++++++++++++++---------- 1 file changed, 81 insertions(+), 24 deletions(-) diff --git a/pkg/config/localdns.go b/pkg/config/localdns.go index d552c806..4f487d5c 100644 --- a/pkg/config/localdns.go +++ b/pkg/config/localdns.go @@ -1,10 +1,12 @@ package config import ( + "bytes" "errors" "fmt" "sort" "strings" + "text/template" ) const ( @@ -90,6 +92,53 @@ func (p *LocalDNSProfile) Enabled() bool { return p != nil && p.Mode == LocalDNSModeRequired } +const aksLocalDNSCorefileTemplate = `health-check.localdns.local:53 { + bind {{.NodeListener}} {{.ClusterListener}} + whoami +} + +{{range .Blocks}}{{.Zone}}:53 { + {{if .LogQueries}}log{{else}}errors{{end}} + bind {{.Listener}} + forward . {{.Upstream}} { + {{if .ForceTCP}}force_tcp + {{end}}policy {{.ForwardPolicy}} + max_concurrent {{.MaxConcurrent}} + } + ready {{.Listener}}:8181 + cache {{.CacheDuration}} { + success 9984 + denial 9984 + {{if .ServeStale}}serve_stale {{.ServeStaleDuration}}s {{.ServeStalePolicy}} + {{end}}servfail 0 + } + loop + prometheus {{$.MetricsAddress}} +} + +{{end}}` + +type localDNSCorefileData struct { + NodeListener string + ClusterListener string + MetricsAddress string + Blocks []localDNSCorefileBlock +} + +type localDNSCorefileBlock struct { + Zone string + Listener string + Upstream string + LogQueries bool + ForceTCP bool + ForwardPolicy string + MaxConcurrent int + CacheDuration int + ServeStale bool + ServeStaleDuration int + ServeStalePolicy string +} + // CorefileTemplate renders AKS LocalDNS policy into an Unbounded Corefile template. func (p *LocalDNSProfile) CorefileTemplate() (string, error) { if p == nil { @@ -111,44 +160,52 @@ func (p *LocalDNSProfile) CorefileTemplate() (string, error) { kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} } - var out strings.Builder - out.WriteString("health-check.localdns.local:53 {\n bind {{ .NodeListenerIP }} {{ .ClusterListenerIP }}\n whoami\n}\n\n") - renderLocalDNSBlocks(&out, vnet, "{{ .NodeListenerIP }}", "VnetDNS") - renderLocalDNSBlocks(&out, kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS") + data := localDNSCorefileData{ + NodeListener: "{{ .NodeListenerIP }}", + ClusterListener: "{{ .ClusterListenerIP }}", + MetricsAddress: "{{ .MetricsAddress }}", + Blocks: append(localDNSCorefileBlocks(vnet, "{{ .NodeListenerIP }}", "VnetDNS"), localDNSCorefileBlocks(kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS")...), + } + tmpl, err := template.New("aks-localdns-corefile").Parse(aksLocalDNSCorefileTemplate) + if err != nil { + return "", fmt.Errorf("parse AKS LocalDNS Corefile template: %w", err) + } + var out bytes.Buffer + if err := tmpl.Execute(&out, data); err != nil { + return "", fmt.Errorf("render AKS LocalDNS Corefile template: %w", err) + } return out.String(), nil } -func renderLocalDNSBlocks(out *strings.Builder, overrides map[string]LocalDNSOverride, listener, defaultDestination string) { +func localDNSCorefileBlocks(overrides map[string]LocalDNSOverride, listener, defaultDestination string) []localDNSCorefileBlock { zones := make([]string, 0, len(overrides)) for zone := range overrides { zones = append(zones, zone) } sort.Strings(zones) + + blocks := make([]localDNSCorefileBlock, 0, len(zones)) for _, zone := range zones { o := withLocalDNSDefaults(overrides[zone], defaultDestination) - fmt.Fprintf(out, "%s:53 {\n", zone) - if o.QueryLogging == "Log" { - out.WriteString(" log\n") - } else { - out.WriteString(" errors\n") - } - fmt.Fprintf(out, " bind %s\n", listener) upstream := "{{ .NodeUpstreamIPsJoined }}" if o.ForwardDestination == "ClusterCoreDNS" { upstream = "{{ .ClusterDNSServiceIP }}" } - fmt.Fprintf(out, " forward . %s {\n", upstream) - if o.Protocol == "ForceTCP" { - out.WriteString(" force_tcp\n") - } - fmt.Fprintf(out, " policy %s\n max_concurrent %d\n }\n", localDNSForwardPolicy(o.ForwardPolicy), o.MaxConcurrent) - fmt.Fprintf(out, " ready %s:8181\n", listener) - fmt.Fprintf(out, " cache %d {\n success 9984\n denial 9984\n", o.CacheDurationInSeconds) - if o.ServeStale != "Disable" { - fmt.Fprintf(out, " serve_stale %ds %s\n", o.ServeStaleDurationInSeconds, strings.ToLower(o.ServeStale)) - } - out.WriteString(" servfail 0\n }\n loop\n prometheus {{ .MetricsAddress }}\n}\n\n") - } + blocks = append(blocks, localDNSCorefileBlock{ + Zone: zone, + Listener: listener, + Upstream: upstream, + LogQueries: o.QueryLogging == "Log", + ForceTCP: o.Protocol == "ForceTCP", + ForwardPolicy: localDNSForwardPolicy(o.ForwardPolicy), + MaxConcurrent: o.MaxConcurrent, + CacheDuration: o.CacheDurationInSeconds, + ServeStale: o.ServeStale != "Disable", + ServeStaleDuration: o.ServeStaleDurationInSeconds, + ServeStalePolicy: strings.ToLower(o.ServeStale), + }) + } + return blocks } func withLocalDNSDefaults(o LocalDNSOverride, destination string) LocalDNSOverride { From e7d2aea8c301d1e33b89b3b24681791380f15ecf Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 29 Jul 2026 05:12:04 +0000 Subject: [PATCH 03/17] test: port AgentBaker LocalDNS policy cases --- pkg/config/localdns_test.go | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/pkg/config/localdns_test.go b/pkg/config/localdns_test.go index 04e92abe..7687ec89 100644 --- a/pkg/config/localdns_test.go +++ b/pkg/config/localdns_test.go @@ -35,6 +35,80 @@ func TestLocalDNSProfileCorefileTemplate(t *testing.T) { } } +func TestLocalDNSProfileCorefileOptions(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + override LocalDNSOverride + want []string + wantAbsent []string + }{ + { + name: "VnetDNS PreferUDP sequential immediate", + override: LocalDNSOverride{QueryLogging: "Log", Protocol: "PreferUDP", ForwardDestination: "VnetDNS", ForwardPolicy: "Sequential", MaxConcurrent: 1000, CacheDurationInSeconds: 30, ServeStaleDurationInSeconds: 60, ServeStale: "Immediate"}, + want: []string{"log", "forward . {{ .NodeUpstreamIPsJoined }}", "policy sequential", "max_concurrent 1000", "cache 30", "serve_stale 60s immediate"}, + wantAbsent: []string{"force_tcp"}, + }, + { + name: "ClusterCoreDNS ForceTCP round robin verify", + override: LocalDNSOverride{QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", ForwardPolicy: "RoundRobin", ServeStale: "Verify"}, + want: []string{"errors", "forward . {{ .ClusterDNSServiceIP }}", "force_tcp", "policy round_robin", "serve_stale 3600s verify"}, + }, + { + name: "random without stale", + override: LocalDNSOverride{ForwardDestination: "ClusterCoreDNS", ForwardPolicy: "Random", ServeStale: "Disable"}, + want: []string{"policy random"}, + wantAbsent: []string{"serve_stale"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + profile := &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + VnetDNSOverrides: map[string]LocalDNSOverride{".": test.override}, + KubeDNSOverrides: map[string]LocalDNSOverride{".": test.override}, + } + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatalf("CorefileTemplate() error = %v", err) + } + for _, want := range test.want { + if !strings.Contains(got, want) { + t.Errorf("CorefileTemplate() missing %q:\n%s", want, got) + } + } + for _, unwanted := range test.wantAbsent { + if strings.Contains(got, unwanted) { + t.Errorf("CorefileTemplate() unexpectedly contains %q:\n%s", unwanted, got) + } + } + }) + } +} + +func TestLocalDNSProfileCorefileZoneOrder(t *testing.T) { + t.Parallel() + + profile := &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{ + "z.example": {}, + ".": {}, + "a.example": {}, + }} + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatal(err) + } + root := strings.Index(got, ".:53 {") + a := strings.Index(got, "a.example:53 {") + z := strings.Index(got, "z.example:53 {") + if root < 0 || a < root || z < a { + t.Fatalf("zones are not deterministic: root=%d a=%d z=%d\n%s", root, a, z, got) + } +} + func TestLocalDNSProfileValidation(t *testing.T) { t.Parallel() @@ -48,6 +122,11 @@ func TestLocalDNSProfileValidation(t *testing.T) { {name: "disabled", profile: &LocalDNSProfile{Mode: LocalDNSModeDisabled}}, {name: "invalid mode", profile: &LocalDNSProfile{Mode: "On"}, wantErr: "mode"}, {name: "invalid protocol", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {Protocol: "TLS"}}}, wantErr: "protocol"}, + {name: "invalid logging", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {QueryLogging: "Debug"}}}, wantErr: "queryLogging"}, + {name: "invalid destination", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ForwardDestination: "External"}}}, wantErr: "forwardDestination"}, + {name: "invalid policy", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ForwardPolicy: "First"}}}, wantErr: "forwardPolicy"}, + {name: "invalid stale", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ServeStale: "Always"}}}, wantErr: "serveStale"}, + {name: "negative cache", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {CacheDurationInSeconds: -1}}}, wantErr: "cacheDurationInSeconds"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { From c9bc8f94a043248b2ece7f965561706edd1d66fc Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 29 Jul 2026 06:00:16 +0000 Subject: [PATCH 04/17] test: use MSI private IP for LocalDNS --- hack/e2e/infra/main.bicep | 1 + hack/e2e/lib/infra.sh | 8 +++++++- hack/e2e/lib/node-join-msi.sh | 4 +++- hack/e2e/lib/validate.sh | 4 +++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/hack/e2e/infra/main.bicep b/hack/e2e/infra/main.bicep index a07e5984..ec44699d 100644 --- a/hack/e2e/infra/main.bicep +++ b/hack/e2e/infra/main.bicep @@ -236,6 +236,7 @@ output clusterFqdn string = aksCluster.properties.fqdn output msiVmName string = vmMsi.outputs.vmName output msiVmIp string = vmMsi.outputs.publicIpAddress +output msiVmPrivateIp string = vmMsi.outputs.privateIpAddress output msiVmPrincipalId string = vmMsi.outputs.principalId output tokenVmName string = vmToken.outputs.vmName diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index cc3d457b..d7ba0224 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -107,7 +107,7 @@ infra_deploy() { --query properties.outputs \ -o json) - local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_principal_id + local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_private_ip msi_vm_principal_id local token_vm_name token_vm_ip token_vm_private_ip offline_vm_name offline_vm_ip offline_vm_private_ip local kubeadm_vm_name kubeadm_vm_ip admin_username @@ -115,6 +115,7 @@ infra_deploy() { cluster_id=$(echo "${outputs}" | jq -r '.clusterId.value') msi_vm_name=$(echo "${outputs}" | jq -r '.msiVmName.value') msi_vm_ip=$(echo "${outputs}" | jq -r '.msiVmIp.value') + msi_vm_private_ip=$(echo "${outputs}" | jq -r '.msiVmPrivateIp.value // ""') msi_vm_principal_id=$(echo "${outputs}" | jq -r '.msiVmPrincipalId.value') token_vm_name=$(echo "${outputs}" | jq -r '.tokenVmName.value') token_vm_ip=$(echo "${outputs}" | jq -r '.tokenVmIp.value') @@ -126,6 +127,10 @@ infra_deploy() { kubeadm_vm_ip=$(echo "${outputs}" | jq -r '.kubeadmVmIp.value') admin_username=$(echo "${outputs}" | jq -r '.adminUsername.value') + if [[ -z "${msi_vm_private_ip}" ]] || ! is_valid_ipv4 "${msi_vm_private_ip}"; then + log_error "Missing or invalid MSI VM private IP from deployment outputs: '${msi_vm_private_ip}'" + return 1 + fi if [[ -z "${token_vm_private_ip}" ]] || ! is_valid_ipv4 "${token_vm_private_ip}"; then log_error "Missing or invalid token VM private IP from deployment outputs: '${token_vm_private_ip}'" return 1 @@ -140,6 +145,7 @@ infra_deploy() { state_set "cluster_id" "${cluster_id}" state_set "msi_vm_name" "${msi_vm_name}" state_set "msi_vm_ip" "${msi_vm_ip}" + state_set "msi_vm_private_ip" "${msi_vm_private_ip}" state_set "msi_vm_principal_id" "${msi_vm_principal_id}" state_set "token_vm_name" "${token_vm_name}" state_set "token_vm_ip" "${token_vm_ip}" diff --git a/hack/e2e/lib/node-join-msi.sh b/hack/e2e/lib/node-join-msi.sh index 053d3593..b90892c0 100644 --- a/hack/e2e/lib/node-join-msi.sh +++ b/hack/e2e/lib/node-join-msi.sh @@ -24,6 +24,8 @@ node_join_msi() { local vm_ip vm_ip="$(state_get msi_vm_ip)" + local vm_private_ip + vm_private_ip="$(state_get msi_vm_private_ip)" local cluster_id cluster_id="$(state_get cluster_id)" local subscription_id @@ -56,7 +58,7 @@ node_join_msi() { "kubelet": { "clusterFQDN": "${server_url}", "caCertData": "${ca_cert_data}", - "nodeIP": "${vm_ip}" + "nodeIP": "${vm_private_ip}" } }, "agent": { diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 44ddd7a3..80686362 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -211,12 +211,13 @@ validate_all_nodes() { local msi_vm_name token_vm_name offline_vm_name kubeadm_vm_name local msi_vm_ip token_vm_ip offline_vm_ip kubeadm_vm_ip - local token_vm_private_ip offline_vm_private_ip + local msi_vm_private_ip token_vm_private_ip offline_vm_private_ip msi_vm_name="$(state_get msi_vm_name)" token_vm_name="$(state_get token_vm_name)" offline_vm_name="$(state_get offline_vm_name)" kubeadm_vm_name="$(state_get kubeadm_vm_name)" msi_vm_ip="$(state_get msi_vm_ip)" + msi_vm_private_ip="$(state_get msi_vm_private_ip)" token_vm_ip="$(state_get token_vm_ip)" offline_vm_ip="$(state_get offline_vm_ip)" kubeadm_vm_ip="$(state_get kubeadm_vm_ip)" @@ -228,6 +229,7 @@ validate_all_nodes() { validate_node_joined "${token_vm_name}" || failed=1 validate_node_joined "${offline_vm_name}" || failed=1 validate_node_joined "${kubeadm_vm_name}" || failed=1 + validate_node_ip "${msi_vm_name}" "${msi_vm_private_ip}" || failed=1 validate_node_ip "${token_vm_name}" "${token_vm_private_ip}" || failed=1 validate_node_ip "${offline_vm_name}" "${offline_vm_private_ip}" || failed=1 validate_npd_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 From 603813409ee56a4fcb7a29f18cf01b5f083543b7 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 30 Jul 2026 03:19:58 +0000 Subject: [PATCH 05/17] test: align LocalDNS behavior with AKS RP --- hack/e2e/lib/validate.sh | 49 +++++++++++++++++++++++++++-- pkg/config/adapter.go | 4 ++- pkg/config/localdns_adapter_test.go | 5 ++- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 80686362..af237e6b 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -170,7 +170,8 @@ REMOTE # --------------------------------------------------------------------------- # validate_localdns_status - Verify nspawn LocalDNS on the selected VM. validate_localdns_status() { - local vm_ip="$1" + local vm_name="$1" + local vm_ip="$2" log_info "Validating LocalDNS on ${vm_ip}..." remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' set -euo pipefail @@ -190,6 +191,50 @@ for chain in OUTPUT PREROUTING; do done done REMOTE + + local state_label + state_label="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/localdns-state}')" + if [[ "${state_label}" != "enabled" ]]; then + log_error "Node ${vm_name} localdns-state=${state_label}, expected enabled" + return 1 + fi + + local cluster_pod="localdns-clusterfirst-${vm_name}" + local default_pod="localdns-default-${vm_name}" + kubectl delete pod "${cluster_pod}" "${default_pod}" --ignore-not-found --wait=false >/dev/null + kubectl run "${cluster_pod}" --image=busybox:1.36 --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"${vm_name}\",\"dnsPolicy\":\"ClusterFirst\"}}" \ + --command -- nslookup kubernetes.default.svc.cluster.local >/dev/null + kubectl run "${default_pod}" --image=busybox:1.36 --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"${vm_name}\",\"dnsPolicy\":\"Default\"}}" \ + --command -- nslookup mcr.microsoft.com >/dev/null + + local pod + for pod in "${cluster_pod}" "${default_pod}"; do + local phase="" + for _ in $(seq 1 60); do + phase="$(kubectl get pod "${pod}" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + [[ "${phase}" == "Succeeded" || "${phase}" == "Failed" ]] && break + sleep 2 + done + if [[ "${phase}" != "Succeeded" ]]; then + kubectl describe pod "${pod}" >&2 || true + kubectl logs "${pod}" >&2 || true + log_error "LocalDNS query pod ${pod} ended in phase ${phase}" + return 1 + fi + done + + if ! kubectl logs "${cluster_pod}" | grep -Eq 'Server:[[:space:]]*169\.254\.10\.11'; then + log_error "ClusterFirst pod did not query LocalDNS cluster listener" + return 1 + fi + if ! kubectl logs "${default_pod}" | grep -Eq 'Server:[[:space:]]*169\.254\.10\.10'; then + log_error "Default-policy pod did not query LocalDNS node listener" + return 1 + fi + kubectl delete pod "${cluster_pod}" "${default_pod}" --ignore-not-found --wait=false >/dev/null + log_success "LocalDNS validation passed on ${vm_ip}" } @@ -233,7 +278,7 @@ validate_all_nodes() { validate_node_ip "${token_vm_name}" "${token_vm_private_ip}" || failed=1 validate_node_ip "${offline_vm_name}" "${offline_vm_private_ip}" || failed=1 validate_npd_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 - validate_localdns_status "${msi_vm_ip}" || failed=1 + validate_localdns_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 validate_npd_status "${token_vm_name}" "${token_vm_ip}" || failed=1 # TODO: re-enable once NPD is included in the upstream Unbounded bootstrap # artifact bundle and resolver used by offline artifact mode. diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index f9e16a30..0edb82ee 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -69,9 +69,11 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { Enabled: profile.Enabled(), CorefileTemplate: corefile, } + state := "disabled" if profile.Enabled() { - labels["kubernetes.azure.com/localdns"] = "enabled" + state = "enabled" } + labels["kubernetes.azure.com/localdns-state"] = state } if cfg.Bootstrap.OfflineArtifacts.Source != "" { diff --git a/pkg/config/localdns_adapter_test.go b/pkg/config/localdns_adapter_test.go index eba36356..20fbdb68 100644 --- a/pkg/config/localdns_adapter_test.go +++ b/pkg/config/localdns_adapter_test.go @@ -28,7 +28,7 @@ func TestToAgentConfigLocalDNS(t *testing.T) { if !strings.Contains(got.LocalDNS.CorefileTemplate, "{{ .ClusterDNSServiceIP }}") { t.Fatalf("CorefileTemplate missing cluster DNS placeholder:\n%s", got.LocalDNS.CorefileTemplate) } - if got.Kubelet.Labels["kubernetes.azure.com/localdns"] != "enabled" { + if got.Kubelet.Labels["kubernetes.azure.com/localdns-state"] != "enabled" { t.Fatalf("LocalDNS node label missing: %#v", got.Kubelet.Labels) } if got.Cluster.ClusterDNS != "10.0.0.10" { @@ -47,4 +47,7 @@ func TestToAgentConfigDisabledLocalDNS(t *testing.T) { if got.LocalDNS == nil || got.LocalDNS.Enabled { t.Fatalf("LocalDNS = %#v, want explicitly disabled", got.LocalDNS) } + if got.Kubelet.Labels["kubernetes.azure.com/localdns-state"] != "disabled" { + t.Fatalf("LocalDNS disabled node label missing: %#v", got.Kubelet.Labels) + } } From 4e0ba316a248f91fcc81c264431e827f0fd265ec Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 30 Jul 2026 19:06:35 +0000 Subject: [PATCH 06/17] refactor: isolate LocalDNS rendering --- pkg/config/config.go | 2 +- .../localdns/assets/localdns.corefile.tmpl | 25 ++ pkg/config/internal/localdns/localdns.go | 229 ++++++++++++++++ pkg/config/internal/localdns/localdns_test.go | 52 ++++ .../testdata/localdns.corefile.golden | 60 +++++ pkg/config/localdns.go | 249 +----------------- pkg/config/localdns_test.go | 2 +- 7 files changed, 376 insertions(+), 243 deletions(-) create mode 100644 pkg/config/internal/localdns/assets/localdns.corefile.tmpl create mode 100644 pkg/config/internal/localdns/localdns.go create mode 100644 pkg/config/internal/localdns/localdns_test.go create mode 100644 pkg/config/internal/localdns/testdata/localdns.corefile.golden diff --git a/pkg/config/config.go b/pkg/config/config.go index 98bdb546..07a8a6d1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -883,7 +883,7 @@ func (c *Config) validate() error { if err := c.Bootstrap.validate(); err != nil { return err } - if err := c.Networking.LocalDNS.validate(); err != nil { + if err := c.Networking.LocalDNS.Validate(); err != nil { return fmt.Errorf("invalid networking.localDNS: %w", err) } diff --git a/pkg/config/internal/localdns/assets/localdns.corefile.tmpl b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl new file mode 100644 index 00000000..cbb86f94 --- /dev/null +++ b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl @@ -0,0 +1,25 @@ +health-check.localdns.local:53 { + bind {{.NodeListener}} {{.ClusterListener}} + whoami +} + +{{range .Blocks}}{{.Zone}}:53 { + {{if .LogQueries}}log{{else}}errors{{end}} + bind {{.Listener}} + forward . {{.Upstream}} { + {{if .ForceTCP}}force_tcp + {{end}}policy {{.ForwardPolicy}} + max_concurrent {{.MaxConcurrent}} + } + ready {{.Listener}}:8181 + cache {{.CacheDuration}} { + success 9984 + denial 9984 + {{if .ServeStale}}serve_stale {{.ServeStaleDuration}}s {{.ServeStalePolicy}} + {{end}}servfail 0 + } + loop + prometheus {{$.MetricsAddress}} +} + +{{end}} \ No newline at end of file diff --git a/pkg/config/internal/localdns/localdns.go b/pkg/config/internal/localdns/localdns.go new file mode 100644 index 00000000..85dd1ce1 --- /dev/null +++ b/pkg/config/internal/localdns/localdns.go @@ -0,0 +1,229 @@ +package localdns + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "sort" + "strings" + "text/template" +) + +const ( + LocalDNSModeRequired = "Required" + LocalDNSModePreferred = "Preferred" + LocalDNSModeDisabled = "Disabled" +) + +// LocalDNSProfile mirrors the AKS node-pool LocalDNS configuration contract. +// Keep rendering behavior aligned with AgentBaker: +// https://github.com/Azure/AgentBaker/blob/62a783b7a967352bb4726e636d22930c9973f4f7/aks-node-controller/parser/templates/localdns.toml.gtpl +type LocalDNSProfile struct { + Mode string `json:"mode"` + VnetDNSOverrides map[string]LocalDNSOverride `json:"vnetDNSOverrides,omitempty"` + KubeDNSOverrides map[string]LocalDNSOverride `json:"kubeDNSOverrides,omitempty"` +} + +// LocalDNSOverride configures one CoreDNS server block. +type LocalDNSOverride struct { + QueryLogging string `json:"queryLogging,omitempty"` + Protocol string `json:"protocol,omitempty"` + ForwardDestination string `json:"forwardDestination,omitempty"` + ForwardPolicy string `json:"forwardPolicy,omitempty"` + MaxConcurrent int `json:"maxConcurrent,omitempty"` + CacheDurationInSeconds int `json:"cacheDurationInSeconds,omitempty"` + ServeStaleDurationInSeconds int `json:"serveStaleDurationInSeconds,omitempty"` + ServeStale string `json:"serveStale,omitempty"` +} + +// Validate verifies the AKS LocalDNS profile values accepted by the renderer. +func (p *LocalDNSProfile) Validate() error { + if p == nil { + return nil + } + if p.Mode != LocalDNSModeRequired && p.Mode != LocalDNSModePreferred && p.Mode != LocalDNSModeDisabled { + return fmt.Errorf("mode must be Required, Preferred, or Disabled") + } + var errs []error + for class, overrides := range map[string]map[string]LocalDNSOverride{ + "vnetDNSOverrides": p.VnetDNSOverrides, + "kubeDNSOverrides": p.KubeDNSOverrides, + } { + for zone, override := range overrides { + if strings.TrimSpace(zone) == "" || strings.ContainsAny(zone, "{} \t\r\n") { + errs = append(errs, fmt.Errorf("%s zone %q is invalid", class, zone)) + } + if err := override.validate(); err != nil { + errs = append(errs, fmt.Errorf("%s[%q]: %w", class, zone, err)) + } + } + } + return errors.Join(errs...) +} + +func (o LocalDNSOverride) validate() error { + var errs []error + if o.QueryLogging != "" && o.QueryLogging != "Error" && o.QueryLogging != "Log" { + errs = append(errs, fmt.Errorf("queryLogging must be Error or Log")) + } + if o.Protocol != "" && o.Protocol != "PreferUDP" && o.Protocol != "ForceTCP" { + errs = append(errs, fmt.Errorf("protocol must be PreferUDP or ForceTCP")) + } + if o.ForwardDestination != "" && o.ForwardDestination != "VnetDNS" && o.ForwardDestination != "ClusterCoreDNS" { + errs = append(errs, fmt.Errorf("forwardDestination must be VnetDNS or ClusterCoreDNS")) + } + if o.ForwardPolicy != "" && o.ForwardPolicy != "Sequential" && o.ForwardPolicy != "RoundRobin" && o.ForwardPolicy != "Random" { + errs = append(errs, fmt.Errorf("forwardPolicy must be Sequential, RoundRobin, or Random")) + } + if o.ServeStale != "" && o.ServeStale != "Disable" && o.ServeStale != "Verify" && o.ServeStale != "Immediate" { + errs = append(errs, fmt.Errorf("serveStale must be Disable, Verify, or Immediate")) + } + for name, value := range map[string]int{ + "maxConcurrent": o.MaxConcurrent, + "cacheDurationInSeconds": o.CacheDurationInSeconds, + "serveStaleDurationInSeconds": o.ServeStaleDurationInSeconds, + } { + if value < 0 { + errs = append(errs, fmt.Errorf("%s must not be negative", name)) + } + } + return errors.Join(errs...) +} + +// Enabled reports whether the profile requires LocalDNS installation. +func (p *LocalDNSProfile) Enabled() bool { + return p != nil && p.Mode == LocalDNSModeRequired +} + +//go:embed assets/localdns.corefile.tmpl +var aksLocalDNSCorefileTemplate string + +type localDNSCorefileData struct { + NodeListener string + ClusterListener string + MetricsAddress string + Blocks []localDNSCorefileBlock +} + +type localDNSCorefileBlock struct { + Zone string + Listener string + Upstream string + LogQueries bool + ForceTCP bool + ForwardPolicy string + MaxConcurrent int + CacheDuration int + ServeStale bool + ServeStaleDuration int + ServeStalePolicy string +} + +// CorefileTemplate renders AKS LocalDNS policy into an Unbounded Corefile template. +func (p *LocalDNSProfile) CorefileTemplate() (string, error) { + if p == nil { + return "", nil + } + if err := p.Validate(); err != nil { + return "", err + } + if !p.Enabled() { + return "", nil + } + + vnet := p.VnetDNSOverrides + if len(vnet) == 0 { + vnet = map[string]LocalDNSOverride{".": {ForwardDestination: "VnetDNS"}} + } + kube := p.KubeDNSOverrides + if len(kube) == 0 { + kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} + } + + data := localDNSCorefileData{ + NodeListener: "{{ .NodeListenerIP }}", + ClusterListener: "{{ .ClusterListenerIP }}", + MetricsAddress: "{{ .MetricsAddress }}", + Blocks: append(localDNSCorefileBlocks(vnet, "{{ .NodeListenerIP }}", "VnetDNS"), localDNSCorefileBlocks(kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS")...), + } + tmpl, err := template.New("aks-localdns-corefile").Parse(aksLocalDNSCorefileTemplate) + if err != nil { + return "", fmt.Errorf("parse AKS LocalDNS Corefile template: %w", err) + } + var out bytes.Buffer + if err := tmpl.Execute(&out, data); err != nil { + return "", fmt.Errorf("render AKS LocalDNS Corefile template: %w", err) + } + return out.String(), nil +} + +func localDNSCorefileBlocks(overrides map[string]LocalDNSOverride, listener, defaultDestination string) []localDNSCorefileBlock { + zones := make([]string, 0, len(overrides)) + for zone := range overrides { + zones = append(zones, zone) + } + sort.Strings(zones) + + blocks := make([]localDNSCorefileBlock, 0, len(zones)) + for _, zone := range zones { + o := withLocalDNSDefaults(overrides[zone], defaultDestination) + upstream := "{{ .NodeUpstreamIPsJoined }}" + if o.ForwardDestination == "ClusterCoreDNS" { + upstream = "{{ .ClusterDNSServiceIP }}" + } + blocks = append(blocks, localDNSCorefileBlock{ + Zone: zone, + Listener: listener, + Upstream: upstream, + LogQueries: o.QueryLogging == "Log", + ForceTCP: o.Protocol == "ForceTCP", + ForwardPolicy: localDNSForwardPolicy(o.ForwardPolicy), + MaxConcurrent: o.MaxConcurrent, + CacheDuration: o.CacheDurationInSeconds, + ServeStale: o.ServeStale != "Disable", + ServeStaleDuration: o.ServeStaleDurationInSeconds, + ServeStalePolicy: strings.ToLower(o.ServeStale), + }) + } + return blocks +} + +func withLocalDNSDefaults(o LocalDNSOverride, destination string) LocalDNSOverride { + if o.QueryLogging == "" { + o.QueryLogging = "Error" + } + if o.Protocol == "" { + o.Protocol = "ForceTCP" + } + if o.ForwardDestination == "" { + o.ForwardDestination = destination + } + if o.ForwardPolicy == "" { + o.ForwardPolicy = "Sequential" + } + if o.MaxConcurrent == 0 { + o.MaxConcurrent = 1000 + } + if o.CacheDurationInSeconds == 0 { + o.CacheDurationInSeconds = 3600 + } + if o.ServeStaleDurationInSeconds == 0 { + o.ServeStaleDurationInSeconds = 3600 + } + if o.ServeStale == "" { + o.ServeStale = "Immediate" + } + return o +} + +func localDNSForwardPolicy(policy string) string { + switch policy { + case "RoundRobin": + return "round_robin" + case "Random": + return "random" + default: + return "sequential" + } +} diff --git a/pkg/config/internal/localdns/localdns_test.go b/pkg/config/internal/localdns/localdns_test.go new file mode 100644 index 00000000..536f1e85 --- /dev/null +++ b/pkg/config/internal/localdns/localdns_test.go @@ -0,0 +1,52 @@ +package localdns + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCorefileTemplateGolden(t *testing.T) { + t.Parallel() + + profile := &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + VnetDNSOverrides: map[string]LocalDNSOverride{ + ".": { + QueryLogging: "Log", Protocol: "PreferUDP", ForwardDestination: "VnetDNS", + ForwardPolicy: "Sequential", MaxConcurrent: 1000, CacheDurationInSeconds: 30, + ServeStaleDurationInSeconds: 60, ServeStale: "Immediate", + }, + "cluster.local": { + QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", + ForwardPolicy: "RoundRobin", MaxConcurrent: 2000, CacheDurationInSeconds: 45, + ServeStale: "Disable", + }, + }, + KubeDNSOverrides: map[string]LocalDNSOverride{ + ".": { + QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", + ForwardPolicy: "Random", MaxConcurrent: 1500, CacheDurationInSeconds: 90, + ServeStaleDurationInSeconds: 120, ServeStale: "Verify", + }, + }, + } + + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatalf("CorefileTemplate() error = %v", err) + } + goldenPath := filepath.Join("testdata", "localdns.corefile.golden") + if os.Getenv("UPDATE_GOLDEN") == "1" { + if err := os.WriteFile(goldenPath, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(goldenPath) + if err != nil { + t.Fatal(err) + } + if got != string(want) { + t.Fatalf("Corefile mismatch (-want +got):\n--- want ---\n%s\n--- got ---\n%s", want, got) + } +} diff --git a/pkg/config/internal/localdns/testdata/localdns.corefile.golden b/pkg/config/internal/localdns/testdata/localdns.corefile.golden new file mode 100644 index 00000000..c1dbcd8d --- /dev/null +++ b/pkg/config/internal/localdns/testdata/localdns.corefile.golden @@ -0,0 +1,60 @@ +health-check.localdns.local:53 { + bind {{ .NodeListenerIP }} {{ .ClusterListenerIP }} + whoami +} + +.:53 { + log + bind {{ .NodeListenerIP }} + forward . {{ .NodeUpstreamIPsJoined }} { + policy sequential + max_concurrent 1000 + } + ready {{ .NodeListenerIP }}:8181 + cache 30 { + success 9984 + denial 9984 + serve_stale 60s immediate + servfail 0 + } + loop + prometheus {{ .MetricsAddress }} +} + +cluster.local:53 { + errors + bind {{ .NodeListenerIP }} + forward . {{ .ClusterDNSServiceIP }} { + force_tcp + policy round_robin + max_concurrent 2000 + } + ready {{ .NodeListenerIP }}:8181 + cache 45 { + success 9984 + denial 9984 + servfail 0 + } + loop + prometheus {{ .MetricsAddress }} +} + +.:53 { + errors + bind {{ .ClusterListenerIP }} + forward . {{ .ClusterDNSServiceIP }} { + force_tcp + policy random + max_concurrent 1500 + } + ready {{ .ClusterListenerIP }}:8181 + cache 90 { + success 9984 + denial 9984 + serve_stale 120s verify + servfail 0 + } + loop + prometheus {{ .MetricsAddress }} +} + diff --git a/pkg/config/localdns.go b/pkg/config/localdns.go index 4f487d5c..0422fbfe 100644 --- a/pkg/config/localdns.go +++ b/pkg/config/localdns.go @@ -1,248 +1,15 @@ package config -import ( - "bytes" - "errors" - "fmt" - "sort" - "strings" - "text/template" -) +import "github.com/Azure/AKSFlexNode/pkg/config/internal/localdns" const ( - LocalDNSModeRequired = "Required" - LocalDNSModePreferred = "Preferred" - LocalDNSModeDisabled = "Disabled" + LocalDNSModeRequired = localdns.LocalDNSModeRequired + LocalDNSModePreferred = localdns.LocalDNSModePreferred + LocalDNSModeDisabled = localdns.LocalDNSModeDisabled ) -// LocalDNSProfile mirrors the AKS node-pool LocalDNS configuration contract. -type LocalDNSProfile struct { - Mode string `json:"mode"` - VnetDNSOverrides map[string]LocalDNSOverride `json:"vnetDNSOverrides,omitempty"` - KubeDNSOverrides map[string]LocalDNSOverride `json:"kubeDNSOverrides,omitempty"` -} - -// LocalDNSOverride configures one CoreDNS server block. -type LocalDNSOverride struct { - QueryLogging string `json:"queryLogging,omitempty"` - Protocol string `json:"protocol,omitempty"` - ForwardDestination string `json:"forwardDestination,omitempty"` - ForwardPolicy string `json:"forwardPolicy,omitempty"` - MaxConcurrent int `json:"maxConcurrent,omitempty"` - CacheDurationInSeconds int `json:"cacheDurationInSeconds,omitempty"` - ServeStaleDurationInSeconds int `json:"serveStaleDurationInSeconds,omitempty"` - ServeStale string `json:"serveStale,omitempty"` -} - -func (p *LocalDNSProfile) validate() error { - if p == nil { - return nil - } - if p.Mode != LocalDNSModeRequired && p.Mode != LocalDNSModePreferred && p.Mode != LocalDNSModeDisabled { - return fmt.Errorf("mode must be Required, Preferred, or Disabled") - } - var errs []error - for class, overrides := range map[string]map[string]LocalDNSOverride{ - "vnetDNSOverrides": p.VnetDNSOverrides, - "kubeDNSOverrides": p.KubeDNSOverrides, - } { - for zone, override := range overrides { - if strings.TrimSpace(zone) == "" || strings.ContainsAny(zone, "{} \t\r\n") { - errs = append(errs, fmt.Errorf("%s zone %q is invalid", class, zone)) - } - if err := override.validate(); err != nil { - errs = append(errs, fmt.Errorf("%s[%q]: %w", class, zone, err)) - } - } - } - return errors.Join(errs...) -} - -func (o LocalDNSOverride) validate() error { - var errs []error - if o.QueryLogging != "" && o.QueryLogging != "Error" && o.QueryLogging != "Log" { - errs = append(errs, fmt.Errorf("queryLogging must be Error or Log")) - } - if o.Protocol != "" && o.Protocol != "PreferUDP" && o.Protocol != "ForceTCP" { - errs = append(errs, fmt.Errorf("protocol must be PreferUDP or ForceTCP")) - } - if o.ForwardDestination != "" && o.ForwardDestination != "VnetDNS" && o.ForwardDestination != "ClusterCoreDNS" { - errs = append(errs, fmt.Errorf("forwardDestination must be VnetDNS or ClusterCoreDNS")) - } - if o.ForwardPolicy != "" && o.ForwardPolicy != "Sequential" && o.ForwardPolicy != "RoundRobin" && o.ForwardPolicy != "Random" { - errs = append(errs, fmt.Errorf("forwardPolicy must be Sequential, RoundRobin, or Random")) - } - if o.ServeStale != "" && o.ServeStale != "Disable" && o.ServeStale != "Verify" && o.ServeStale != "Immediate" { - errs = append(errs, fmt.Errorf("serveStale must be Disable, Verify, or Immediate")) - } - for name, value := range map[string]int{ - "maxConcurrent": o.MaxConcurrent, - "cacheDurationInSeconds": o.CacheDurationInSeconds, - "serveStaleDurationInSeconds": o.ServeStaleDurationInSeconds, - } { - if value < 0 { - errs = append(errs, fmt.Errorf("%s must not be negative", name)) - } - } - return errors.Join(errs...) -} - -// Enabled reports whether the profile requires LocalDNS installation. -func (p *LocalDNSProfile) Enabled() bool { - return p != nil && p.Mode == LocalDNSModeRequired -} - -const aksLocalDNSCorefileTemplate = `health-check.localdns.local:53 { - bind {{.NodeListener}} {{.ClusterListener}} - whoami -} - -{{range .Blocks}}{{.Zone}}:53 { - {{if .LogQueries}}log{{else}}errors{{end}} - bind {{.Listener}} - forward . {{.Upstream}} { - {{if .ForceTCP}}force_tcp - {{end}}policy {{.ForwardPolicy}} - max_concurrent {{.MaxConcurrent}} - } - ready {{.Listener}}:8181 - cache {{.CacheDuration}} { - success 9984 - denial 9984 - {{if .ServeStale}}serve_stale {{.ServeStaleDuration}}s {{.ServeStalePolicy}} - {{end}}servfail 0 - } - loop - prometheus {{$.MetricsAddress}} -} - -{{end}}` - -type localDNSCorefileData struct { - NodeListener string - ClusterListener string - MetricsAddress string - Blocks []localDNSCorefileBlock -} - -type localDNSCorefileBlock struct { - Zone string - Listener string - Upstream string - LogQueries bool - ForceTCP bool - ForwardPolicy string - MaxConcurrent int - CacheDuration int - ServeStale bool - ServeStaleDuration int - ServeStalePolicy string -} - -// CorefileTemplate renders AKS LocalDNS policy into an Unbounded Corefile template. -func (p *LocalDNSProfile) CorefileTemplate() (string, error) { - if p == nil { - return "", nil - } - if err := p.validate(); err != nil { - return "", err - } - if !p.Enabled() { - return "", nil - } - - vnet := p.VnetDNSOverrides - if len(vnet) == 0 { - vnet = map[string]LocalDNSOverride{".": {ForwardDestination: "VnetDNS"}} - } - kube := p.KubeDNSOverrides - if len(kube) == 0 { - kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} - } - - data := localDNSCorefileData{ - NodeListener: "{{ .NodeListenerIP }}", - ClusterListener: "{{ .ClusterListenerIP }}", - MetricsAddress: "{{ .MetricsAddress }}", - Blocks: append(localDNSCorefileBlocks(vnet, "{{ .NodeListenerIP }}", "VnetDNS"), localDNSCorefileBlocks(kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS")...), - } - tmpl, err := template.New("aks-localdns-corefile").Parse(aksLocalDNSCorefileTemplate) - if err != nil { - return "", fmt.Errorf("parse AKS LocalDNS Corefile template: %w", err) - } - var out bytes.Buffer - if err := tmpl.Execute(&out, data); err != nil { - return "", fmt.Errorf("render AKS LocalDNS Corefile template: %w", err) - } - return out.String(), nil -} - -func localDNSCorefileBlocks(overrides map[string]LocalDNSOverride, listener, defaultDestination string) []localDNSCorefileBlock { - zones := make([]string, 0, len(overrides)) - for zone := range overrides { - zones = append(zones, zone) - } - sort.Strings(zones) - - blocks := make([]localDNSCorefileBlock, 0, len(zones)) - for _, zone := range zones { - o := withLocalDNSDefaults(overrides[zone], defaultDestination) - upstream := "{{ .NodeUpstreamIPsJoined }}" - if o.ForwardDestination == "ClusterCoreDNS" { - upstream = "{{ .ClusterDNSServiceIP }}" - } - blocks = append(blocks, localDNSCorefileBlock{ - Zone: zone, - Listener: listener, - Upstream: upstream, - LogQueries: o.QueryLogging == "Log", - ForceTCP: o.Protocol == "ForceTCP", - ForwardPolicy: localDNSForwardPolicy(o.ForwardPolicy), - MaxConcurrent: o.MaxConcurrent, - CacheDuration: o.CacheDurationInSeconds, - ServeStale: o.ServeStale != "Disable", - ServeStaleDuration: o.ServeStaleDurationInSeconds, - ServeStalePolicy: strings.ToLower(o.ServeStale), - }) - } - return blocks -} - -func withLocalDNSDefaults(o LocalDNSOverride, destination string) LocalDNSOverride { - if o.QueryLogging == "" { - o.QueryLogging = "Error" - } - if o.Protocol == "" { - o.Protocol = "ForceTCP" - } - if o.ForwardDestination == "" { - o.ForwardDestination = destination - } - if o.ForwardPolicy == "" { - o.ForwardPolicy = "Sequential" - } - if o.MaxConcurrent == 0 { - o.MaxConcurrent = 1000 - } - if o.CacheDurationInSeconds == 0 { - o.CacheDurationInSeconds = 3600 - } - if o.ServeStaleDurationInSeconds == 0 { - o.ServeStaleDurationInSeconds = 3600 - } - if o.ServeStale == "" { - o.ServeStale = "Immediate" - } - return o -} +// LocalDNSProfile is the AKS node-pool LocalDNS configuration contract. +type LocalDNSProfile = localdns.LocalDNSProfile -func localDNSForwardPolicy(policy string) string { - switch policy { - case "RoundRobin": - return "round_robin" - case "Random": - return "random" - default: - return "sequential" - } -} +// LocalDNSOverride configures one AKS LocalDNS server block. +type LocalDNSOverride = localdns.LocalDNSOverride diff --git a/pkg/config/localdns_test.go b/pkg/config/localdns_test.go index 7687ec89..81524ed1 100644 --- a/pkg/config/localdns_test.go +++ b/pkg/config/localdns_test.go @@ -131,7 +131,7 @@ func TestLocalDNSProfileValidation(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - err := test.profile.validate() + err := test.profile.Validate() if test.wantErr == "" && err != nil { t.Fatalf("validate() error = %v", err) } From 20eb62a1647eb491377d33d53a614cd479b55658 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 30 Jul 2026 19:44:16 +0000 Subject: [PATCH 07/17] test: validate LocalDNS disable through repave --- hack/e2e/lib/upgrade-drift.sh | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/hack/e2e/lib/upgrade-drift.sh b/hack/e2e/lib/upgrade-drift.sh index f1be588f..c2c3970a 100644 --- a/hack/e2e/lib/upgrade-drift.sh +++ b/hack/e2e/lib/upgrade-drift.sh @@ -209,11 +209,60 @@ upgrade_drift_mode() { log_success "${mode} controller machine repave passed in $(timer_elapsed "${start}")s" } +localdns_disable_repave_msi() { + log_section "LocalDNS Disable Repave (MSI node)" + local desired_version settings_version vm_name vm_ip snapshot old_active_machine old_state old_version old_settings_version old_node_uid + desired_version="$(_cluster_current_kubernetes_version)" + settings_version="localdns-disabled-$(date +%s)" + vm_name="$(_mode_vm_name msi)" + vm_ip="$(_mode_vm_ip msi)" + + snapshot="$(_remote_active_machine_snapshot "${vm_ip}")" + IFS='|' read -r old_active_machine old_state old_version old_settings_version <<<"${snapshot}" + old_node_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" + + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail +config=/etc/aks-flex-node/config.json +tmp=$(mktemp /etc/aks-flex-node/config.json.XXXXXX) +jq '.networking.localDNS.mode = "Disabled"' "${config}" > "${tmp}" +chmod 0600 "${tmp}" +mv "${tmp}" "${config}" +systemctl restart aks-flex-node-agent.service +systemctl is-active --quiet aks-flex-node-agent.service +REMOTE + + _trigger_mode_repave msi "${desired_version}" "${settings_version}" + _wait_for_mode_repave msi "${desired_version}" "${settings_version}" "${old_active_machine}" "${old_node_uid}" + + local state_label cluster_dns + state_label="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/localdns-state}')" + cluster_dns="$(kubectl -n kube-system get service kube-dns -o jsonpath='{.spec.clusterIP}')" + if [[ "${state_label}" != "disabled" ]]; then + log_error "Node ${vm_name} localdns-state=${state_label}, expected disabled" + return 1 + fi + + remote_exec "${vm_ip}" "CLUSTER_DNS=${cluster_dns} sudo -E bash -s" <<'REMOTE' +set -euo pipefail +machine=$(machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}') +test -n "${machine}" +! systemd-run --quiet --pipe --wait --machine="${machine}" systemctl cat localdns.service >/dev/null 2>&1 +! ip link show localdns >/dev/null 2>&1 +! iptables -w -t raw -S | grep -q 'unbounded-localdns: skip conntrack' +systemd-run --quiet --pipe --wait --machine="${machine}" \ + grep -q -- "--cluster-dns=${CLUSTER_DNS}" /etc/systemd/system/kubelet.service.d/20-node-config.conf +REMOTE + + log_success "MSI LocalDNS disable-through-repave validation passed" +} + upgrade_drift_all() { log_section "Controller Machine Repave (all modes)" upgrade_drift_mode msi upgrade_drift_mode token upgrade_drift_mode kubeadm + localdns_disable_repave_msi } upgrade_drift_msi() { upgrade_drift_mode msi; } From e6d87dbac24e6a1bb311fac5c3395d825c3df913 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 4 Aug 2026 20:53:46 +0000 Subject: [PATCH 08/17] build: bump Unbounded for nspawn LocalDNS --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f7009d23..56079775 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.2 + github.com/Azure/unbounded v0.2.3-0.20260804212734-7fc47a19d0c3 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index dd35efd4..3d70b02f 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.2 h1:uu5hYj20UBbrSAlf4qnDiMTGaR0xxsYnkdbTWTdBHX8= -github.com/Azure/unbounded v0.2.2/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw= +github.com/Azure/unbounded v0.2.3-0.20260804212734-7fc47a19d0c3 h1:QNdVeXxDMQJaVArB3cRuuZY5qd2Ww13EudPaZ3AUyf0= +github.com/Azure/unbounded v0.2.3-0.20260804212734-7fc47a19d0c3/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 9e77ac8ce7fca641a27ee2ad8564463f0ce56ed1 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 4 Aug 2026 21:49:40 +0000 Subject: [PATCH 09/17] test: prepare LocalDNS E2E host resolver --- hack/e2e/lib/node-join-msi.sh | 62 ++++++++++++++++++++++++++++++++++- hack/e2e/lib/validate.sh | 14 +++++--- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/hack/e2e/lib/node-join-msi.sh b/hack/e2e/lib/node-join-msi.sh index b90892c0..6a94715e 100644 --- a/hack/e2e/lib/node-join-msi.sh +++ b/hack/e2e/lib/node-join-msi.sh @@ -14,6 +14,63 @@ readonly _E2E_NODE_JOIN_MSI_LOADED=1 # shellcheck disable=SC1091 source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +# --------------------------------------------------------------------------- +# prepare_localdns_host_resolver - Remove DHCP search/routing domains. +# +# Unbounded intentionally rejects systemd-resolved split-DNS layouts because +# flattening per-domain routing into one LocalDNS upstream list changes DNS +# semantics. Azure's Ubuntu image supplies a DHCP search domain by default, so +# put the disposable E2E host into the supported single-upstream layout before +# LocalDNS preflight runs. +# --------------------------------------------------------------------------- +prepare_localdns_host_resolver() { + local vm_ip="$1" + + log_info "Configuring the MSI host resolver for LocalDNS..." + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail + +interface="$(ip -4 route show default | awk 'NR == 1 { print $5 }')" +if [[ ! "${interface}" =~ ^[a-zA-Z0-9_.:-]+$ ]]; then + echo "could not determine a safe primary interface name: ${interface}" >&2 + exit 1 +fi + +cat > /etc/netplan/99-aks-flex-localdns.yaml <&2 + resolvectl domain >&2 + exit 1 +fi + +resolvectl domain +resolvectl dns "${interface}" +REMOTE +} + # --------------------------------------------------------------------------- # node_join_msi - Join the MSI VM # --------------------------------------------------------------------------- @@ -107,7 +164,10 @@ node_join_msi() { } EOF - # Step 2: Publish the AKS Machine goal and deploy the agent. + # Step 2: Put systemd-resolved into the layout supported by LocalDNS. + prepare_localdns_host_resolver "${vm_ip}" + + # Step 3: Publish the AKS Machine goal and deploy the agent. ensure_flex_controller machine_configmap_upsert "$(state_get msi_vm_name)" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}" _deploy_and_start_agent "${vm_ip}" "${config_file}" "aks-flex-node-msi" diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index af237e6b..ac299d65 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -178,15 +178,19 @@ set -euo pipefail machine=$(sudo machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}') test -n "${machine}" sudo systemd-run --quiet --pipe --wait --machine="${machine}" systemctl is-active --quiet localdns.service -sudo systemd-run --quiet --pipe --wait --machine="${machine}" grep -qx 'nameserver 169.254.10.10' /etc/resolv.conf +sudo systemd-run --quiet --pipe --wait --machine="${machine}" \ + grep -qx 'nameserver 169.254.10.10' /etc/unbounded/localdns/resolv.conf +sudo systemd-run --quiet --pipe --wait --machine="${machine}" \ + systemctl cat kubelet.service | grep -q -- '--resolv-conf=/etc/unbounded/localdns/resolv.conf' sudo ip address show dev localdns | grep -q '169.254.10.10/32' sudo ip address show dev localdns | grep -q '169.254.10.11/32' -for chain in OUTPUT PREROUTING; do +for chain in output prerouting; do + rules="$(sudo nft list chain ip unbounded_localdns "${chain}")" for address in 169.254.10.10 169.254.10.11; do for protocol in tcp udp; do - sudo iptables -w -t raw -C "${chain}" -m comment \ - --comment 'unbounded-localdns: skip conntrack' \ - -p "${protocol}" -d "${address}" --dport 53 -j NOTRACK + grep -Fq \ + "ip daddr ${address} ${protocol} dport 53 notrack comment \"unbounded-localdns: skip conntrack\"" \ + <<<"${rules}" done done done From d580a1af2cb82f17abb79c5d1bf1e3ffd1354a16 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 20:56:24 +0000 Subject: [PATCH 10/17] chore: bump Unbounded to main --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 56079775..1911ccc7 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,14 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260804212734-7fc47a19d0c3 + github.com/Azure/unbounded v0.2.3-0.20260807193935-99f997bf27b6 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.2 - k8s.io/api v0.36.2 - k8s.io/apimachinery v0.36.2 - k8s.io/client-go v0.36.2 + k8s.io/api v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/client-go v0.36.3 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 ) @@ -101,12 +101,12 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.36.0 // indirect + k8s.io/apiextensions-apiserver v0.36.3 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 // indirect oras.land/oras-go/v2 v2.6.2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 3d70b02f..61fa40a4 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260804212734-7fc47a19d0c3 h1:QNdVeXxDMQJaVArB3cRuuZY5qd2Ww13EudPaZ3AUyf0= -github.com/Azure/unbounded v0.2.3-0.20260804212734-7fc47a19d0c3/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw= +github.com/Azure/unbounded v0.2.3-0.20260807193935-99f997bf27b6 h1:oT3ocXgKPW985AIlasqzEiKS02oMyWiDVEMusn3XaEI= +github.com/Azure/unbounded v0.2.3-0.20260807193935-99f997bf27b6/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= @@ -348,14 +348,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= -k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= -k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= -k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= -k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= -k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= -k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= -k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= +k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0= +k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 h1:Sztf7ESG9tAXRW/ACJZjrj5jhdOUqS2KFRQT+CTvu78= @@ -370,7 +370,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= From 39e2d02c36fe191383b287535b19be24d342df76 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 20:58:41 +0000 Subject: [PATCH 11/17] chore: pin Unbounded v0.2.3-alpha.3 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1911ccc7..6f017b83 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260807193935-99f997bf27b6 + github.com/Azure/unbounded v0.2.3-alpha.3 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 61fa40a4..7e9dd9a4 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260807193935-99f997bf27b6 h1:oT3ocXgKPW985AIlasqzEiKS02oMyWiDVEMusn3XaEI= -github.com/Azure/unbounded v0.2.3-0.20260807193935-99f997bf27b6/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= +github.com/Azure/unbounded v0.2.3-alpha.3 h1:grxHFtQpulYHSFRusRTPZl//bPrHO9lifsJrjcFVrxo= +github.com/Azure/unbounded v0.2.3-alpha.3/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 4c117b40e9502f00f8b1ed96043d9132297da587 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 21:11:03 +0000 Subject: [PATCH 12/17] fix: align LocalDNS Corefile with AgentBaker --- pkg/config/adapter.go | 1 + .../localdns/assets/localdns.corefile.tmpl | 20 +++++++++- pkg/config/internal/localdns/localdns.go | 20 +++++++++- pkg/config/internal/localdns/localdns_test.go | 37 +++++++++++++++++++ .../testdata/localdns.corefile.golden | 28 ++++++++++++++ pkg/config/localdns_adapter_test.go | 5 +++ 6 files changed, 109 insertions(+), 2 deletions(-) diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index 0edb82ee..bd709fc8 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -67,6 +67,7 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { corefile, _ := profile.CorefileTemplate() // Config validation runs before adaptation. ac.LocalDNS = &agentconfig.AgentLocalDNSConfig{ Enabled: profile.Enabled(), + RequiredPlugins: []string{"nsid", "template"}, CorefileTemplate: corefile, } state := "disabled" diff --git a/pkg/config/internal/localdns/assets/localdns.corefile.tmpl b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl index cbb86f94..5bfc65fd 100644 --- a/pkg/config/internal/localdns/assets/localdns.corefile.tmpl +++ b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl @@ -1,3 +1,12 @@ +# This Corefile follows the AKS AgentBaker LocalDNS policy with adaptations for +# the Unbounded nspawn runtime: +# - Node DNS forwards to resolver addresses discovered from the physical host +# instead of assuming the Azure platform DNS address. +# - The metrics endpoint binds a runtime-resolved node address instead of :9253. +# - The AgentBaker critical-FQDN hosts file is omitted because Unbounded does not +# run the AKS hosts-file refresher. +# Listener, upstream, cluster DNS, and metrics placeholders are resolved by +# Unbounded after AKS Flex Node renders the profile-specific server blocks. health-check.localdns.local:53 { bind {{.NodeListener}} {{.ClusterListener}} whoami @@ -19,7 +28,16 @@ health-check.localdns.local:53 { {{end}}servfail 0 } loop + nsid {{.NSID}} prometheus {{$.MetricsAddress}} -} +{{if .IsRootDomain}} template ANY ANY internal.cloudapp.net { + match "^(?:[^.]+\.){4,}internal\.cloudapp\.net\.$" + rcode NXDOMAIN + fallthrough + } + template ANY ANY reddog.microsoft.com { + rcode NXDOMAIN + } +{{end}}} {{end}} \ No newline at end of file diff --git a/pkg/config/internal/localdns/localdns.go b/pkg/config/internal/localdns/localdns.go index 85dd1ce1..dabd8f90 100644 --- a/pkg/config/internal/localdns/localdns.go +++ b/pkg/config/internal/localdns/localdns.go @@ -108,8 +108,10 @@ type localDNSCorefileData struct { type localDNSCorefileBlock struct { Zone string + IsRootDomain bool Listener string Upstream string + NSID string LogQueries bool ForceTCP bool ForwardPolicy string @@ -141,6 +143,9 @@ func (p *LocalDNSProfile) CorefileTemplate() (string, error) { kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} } + // Rendering happens in two stages. This pass translates the AKS profile into + // a Corefile template; Unbounded later resolves these literal placeholders + // from machine-specific listener, upstream, cluster DNS, and metrics values. data := localDNSCorefileData{ NodeListener: "{{ .NodeListenerIP }}", ClusterListener: "{{ .ClusterListenerIP }}", @@ -168,14 +173,27 @@ func localDNSCorefileBlocks(overrides map[string]LocalDNSOverride, listener, def blocks := make([]localDNSCorefileBlock, 0, len(zones)) for _, zone := range zones { o := withLocalDNSDefaults(overrides[zone], defaultDestination) + isRootDomain := zone == "." + forwardToClusterDNS := o.ForwardDestination == "ClusterCoreDNS" || strings.HasSuffix(zone, "cluster.local") + // AgentBaker always sends the VnetDNS root zone to the node's DNS path. + if defaultDestination == "VnetDNS" && isRootDomain { + forwardToClusterDNS = false + } + upstream := "{{ .NodeUpstreamIPsJoined }}" - if o.ForwardDestination == "ClusterCoreDNS" { + if forwardToClusterDNS { upstream = "{{ .ClusterDNSServiceIP }}" } + nsid := "localdns" + if defaultDestination == "ClusterCoreDNS" { + nsid = "localdns-pod" + } blocks = append(blocks, localDNSCorefileBlock{ Zone: zone, + IsRootDomain: isRootDomain, Listener: listener, Upstream: upstream, + NSID: nsid, LogQueries: o.QueryLogging == "Log", ForceTCP: o.Protocol == "ForceTCP", ForwardPolicy: localDNSForwardPolicy(o.ForwardPolicy), diff --git a/pkg/config/internal/localdns/localdns_test.go b/pkg/config/internal/localdns/localdns_test.go index 536f1e85..b6782c39 100644 --- a/pkg/config/internal/localdns/localdns_test.go +++ b/pkg/config/internal/localdns/localdns_test.go @@ -50,3 +50,40 @@ func TestCorefileTemplateGolden(t *testing.T) { t.Fatalf("Corefile mismatch (-want +got):\n--- want ---\n%s\n--- got ---\n%s", want, got) } } + +func TestLocalDNSCorefileBlocksAgentBakerRouting(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + zone string + destination string + defaultDestination string + wantUpstream string + wantNSID string + }{ + {name: "VnetDNS root stays on node DNS", zone: ".", destination: "ClusterCoreDNS", defaultDestination: "VnetDNS", wantUpstream: "{{ .NodeUpstreamIPsJoined }}", wantNSID: "localdns"}, + {name: "VnetDNS cluster suffix uses cluster DNS", zone: "svc.cluster.local", destination: "VnetDNS", defaultDestination: "VnetDNS", wantUpstream: "{{ .ClusterDNSServiceIP }}", wantNSID: "localdns"}, + {name: "KubeDNS cluster suffix uses cluster DNS", zone: "svc.cluster.local", destination: "VnetDNS", defaultDestination: "ClusterCoreDNS", wantUpstream: "{{ .ClusterDNSServiceIP }}", wantNSID: "localdns-pod"}, + {name: "KubeDNS custom zone honors VnetDNS", zone: "example.com", destination: "VnetDNS", defaultDestination: "ClusterCoreDNS", wantUpstream: "{{ .NodeUpstreamIPsJoined }}", wantNSID: "localdns-pod"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + blocks := localDNSCorefileBlocks(map[string]LocalDNSOverride{ + test.zone: {ForwardDestination: test.destination}, + }, "listener", test.defaultDestination) + if len(blocks) != 1 { + t.Fatalf("localDNSCorefileBlocks() returned %d blocks, want 1", len(blocks)) + } + if blocks[0].Upstream != test.wantUpstream { + t.Errorf("Upstream = %q, want %q", blocks[0].Upstream, test.wantUpstream) + } + if blocks[0].NSID != test.wantNSID { + t.Errorf("NSID = %q, want %q", blocks[0].NSID, test.wantNSID) + } + }) + } +} diff --git a/pkg/config/internal/localdns/testdata/localdns.corefile.golden b/pkg/config/internal/localdns/testdata/localdns.corefile.golden index c1dbcd8d..44b61cde 100644 --- a/pkg/config/internal/localdns/testdata/localdns.corefile.golden +++ b/pkg/config/internal/localdns/testdata/localdns.corefile.golden @@ -1,3 +1,12 @@ +# This Corefile follows the AKS AgentBaker LocalDNS policy with adaptations for +# the Unbounded nspawn runtime: +# - Node DNS forwards to resolver addresses discovered from the physical host +# instead of assuming the Azure platform DNS address. +# - The metrics endpoint binds a runtime-resolved node address instead of :9253. +# - The AgentBaker critical-FQDN hosts file is omitted because Unbounded does not +# run the AKS hosts-file refresher. +# Listener, upstream, cluster DNS, and metrics placeholders are resolved by +# Unbounded after AKS Flex Node renders the profile-specific server blocks. health-check.localdns.local:53 { bind {{ .NodeListenerIP }} {{ .ClusterListenerIP }} whoami @@ -18,7 +27,16 @@ health-check.localdns.local:53 { servfail 0 } loop + nsid localdns prometheus {{ .MetricsAddress }} + template ANY ANY internal.cloudapp.net { + match "^(?:[^.]+\.){4,}internal\.cloudapp\.net\.$" + rcode NXDOMAIN + fallthrough + } + template ANY ANY reddog.microsoft.com { + rcode NXDOMAIN + } } cluster.local:53 { @@ -36,6 +54,7 @@ cluster.local:53 { servfail 0 } loop + nsid localdns prometheus {{ .MetricsAddress }} } @@ -55,6 +74,15 @@ cluster.local:53 { servfail 0 } loop + nsid localdns-pod prometheus {{ .MetricsAddress }} + template ANY ANY internal.cloudapp.net { + match "^(?:[^.]+\.){4,}internal\.cloudapp\.net\.$" + rcode NXDOMAIN + fallthrough + } + template ANY ANY reddog.microsoft.com { + rcode NXDOMAIN + } } diff --git a/pkg/config/localdns_adapter_test.go b/pkg/config/localdns_adapter_test.go index 20fbdb68..0be0025e 100644 --- a/pkg/config/localdns_adapter_test.go +++ b/pkg/config/localdns_adapter_test.go @@ -28,6 +28,11 @@ func TestToAgentConfigLocalDNS(t *testing.T) { if !strings.Contains(got.LocalDNS.CorefileTemplate, "{{ .ClusterDNSServiceIP }}") { t.Fatalf("CorefileTemplate missing cluster DNS placeholder:\n%s", got.LocalDNS.CorefileTemplate) } + for _, plugin := range []string{"nsid", "template"} { + if !strings.Contains(strings.Join(got.LocalDNS.RequiredPlugins, ","), plugin) { + t.Errorf("RequiredPlugins = %v, want %q", got.LocalDNS.RequiredPlugins, plugin) + } + } if got.Kubelet.Labels["kubernetes.azure.com/localdns-state"] != "enabled" { t.Fatalf("LocalDNS node label missing: %#v", got.Kubelet.Labels) } From e0127b984b0c6b18cc398c3d8beb00f530c33d7e Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 21:18:50 +0000 Subject: [PATCH 13/17] fix: refine LocalDNS compatibility policy --- hack/e2e/lib/upgrade-drift.sh | 7 +------ hack/e2e/lib/validate.sh | 7 ------- pkg/config/adapter.go | 15 ++------------ .../localdns/assets/localdns.corefile.tmpl | 13 ++++-------- pkg/config/internal/localdns/localdns.go | 11 ++++++++-- pkg/config/internal/localdns/localdns_test.go | 2 +- .../testdata/localdns.corefile.golden | 20 +++---------------- pkg/config/localdns_adapter_test.go | 8 +------- pkg/config/localdns_test.go | 17 +++++++++++----- 9 files changed, 33 insertions(+), 67 deletions(-) diff --git a/hack/e2e/lib/upgrade-drift.sh b/hack/e2e/lib/upgrade-drift.sh index c2c3970a..609e536d 100644 --- a/hack/e2e/lib/upgrade-drift.sh +++ b/hack/e2e/lib/upgrade-drift.sh @@ -235,13 +235,8 @@ REMOTE _trigger_mode_repave msi "${desired_version}" "${settings_version}" _wait_for_mode_repave msi "${desired_version}" "${settings_version}" "${old_active_machine}" "${old_node_uid}" - local state_label cluster_dns - state_label="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/localdns-state}')" + local cluster_dns cluster_dns="$(kubectl -n kube-system get service kube-dns -o jsonpath='{.spec.clusterIP}')" - if [[ "${state_label}" != "disabled" ]]; then - log_error "Node ${vm_name} localdns-state=${state_label}, expected disabled" - return 1 - fi remote_exec "${vm_ip}" "CLUSTER_DNS=${cluster_dns} sudo -E bash -s" <<'REMOTE' set -euo pipefail diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index ac299d65..92a32462 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -196,13 +196,6 @@ for chain in output prerouting; do done REMOTE - local state_label - state_label="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/localdns-state}')" - if [[ "${state_label}" != "enabled" ]]; then - log_error "Node ${vm_name} localdns-state=${state_label}, expected enabled" - return 1 - fi - local cluster_pod="localdns-clusterfirst-${vm_name}" local default_pod="localdns-default-${vm_name}" kubectl delete pod "${cluster_pod}" "${default_pod}" --ignore-not-found --wait=false >/dev/null diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index bd709fc8..698d8f48 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log/slog" - "maps" agentconfig "github.com/Azure/unbounded/pkg/agent/config" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -27,11 +26,6 @@ const ( // // cfg.Node.Kubelet.ClusterFQDN and cfg.Node.Kubelet.CACertData must be populated. func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { - labels := maps.Clone(cfg.Node.Labels) - if labels == nil { - labels = map[string]string{} - } - ac := &agentconfig.AgentConfig{ MachineName: machineName, NodeName: cfg.Agent.NodeName, @@ -46,7 +40,7 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { Kubelet: agentconfig.AgentKubeletConfig{ ApiServer: cfg.APIServerURL(), NodeIP: cfg.Node.Kubelet.NodeIP, - Labels: labels, + Labels: cfg.Node.Labels, RegisterWithTaints: cfg.Node.Taints, }, CRI: agentconfig.CRIConfig{ @@ -67,14 +61,9 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { corefile, _ := profile.CorefileTemplate() // Config validation runs before adaptation. ac.LocalDNS = &agentconfig.AgentLocalDNSConfig{ Enabled: profile.Enabled(), - RequiredPlugins: []string{"nsid", "template"}, + RequiredPlugins: []string{"log", "nsid"}, CorefileTemplate: corefile, } - state := "disabled" - if profile.Enabled() { - state = "enabled" - } - labels["kubernetes.azure.com/localdns-state"] = state } if cfg.Bootstrap.OfflineArtifacts.Source != "" { diff --git a/pkg/config/internal/localdns/assets/localdns.corefile.tmpl b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl index 5bfc65fd..c4c54bab 100644 --- a/pkg/config/internal/localdns/assets/localdns.corefile.tmpl +++ b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl @@ -5,6 +5,9 @@ # - The metrics endpoint binds a runtime-resolved node address instead of :9253. # - The AgentBaker critical-FQDN hosts file is omitted because Unbounded does not # run the AKS hosts-file refresher. +# - Azure VM suffix guards for internal.cloudapp.net and reddog.microsoft.com +# are omitted because Flex Nodes may use non-Azure DNS where those names must +# be resolved according to the host's upstream policy. # Listener, upstream, cluster DNS, and metrics placeholders are resolved by # Unbounded after AKS Flex Node renders the profile-specific server blocks. health-check.localdns.local:53 { @@ -30,14 +33,6 @@ health-check.localdns.local:53 { loop nsid {{.NSID}} prometheus {{$.MetricsAddress}} -{{if .IsRootDomain}} template ANY ANY internal.cloudapp.net { - match "^(?:[^.]+\.){4,}internal\.cloudapp\.net\.$" - rcode NXDOMAIN - fallthrough - } - template ANY ANY reddog.microsoft.com { - rcode NXDOMAIN - } -{{end}}} +} {{end}} \ No newline at end of file diff --git a/pkg/config/internal/localdns/localdns.go b/pkg/config/internal/localdns/localdns.go index dabd8f90..e414ee72 100644 --- a/pkg/config/internal/localdns/localdns.go +++ b/pkg/config/internal/localdns/localdns.go @@ -57,6 +57,12 @@ func (p *LocalDNSProfile) Validate() error { if err := override.validate(); err != nil { errs = append(errs, fmt.Errorf("%s[%q]: %w", class, zone, err)) } + if class == "vnetDNSOverrides" && zone == "." && override.ForwardDestination == "ClusterCoreDNS" { + errs = append(errs, fmt.Errorf("%s[%q]: forwardDestination must not be ClusterCoreDNS", class, zone)) + } + if strings.HasSuffix(zone, "cluster.local") && override.ForwardDestination == "VnetDNS" { + errs = append(errs, fmt.Errorf("%s[%q]: forwardDestination must not be VnetDNS", class, zone)) + } } } return errors.Join(errs...) @@ -79,6 +85,9 @@ func (o LocalDNSOverride) validate() error { if o.ServeStale != "" && o.ServeStale != "Disable" && o.ServeStale != "Verify" && o.ServeStale != "Immediate" { errs = append(errs, fmt.Errorf("serveStale must be Disable, Verify, or Immediate")) } + if (o.Protocol == "" || o.Protocol == "ForceTCP") && o.ServeStale == "Verify" { + errs = append(errs, fmt.Errorf("serveStale Verify requires protocol PreferUDP")) + } for name, value := range map[string]int{ "maxConcurrent": o.MaxConcurrent, "cacheDurationInSeconds": o.CacheDurationInSeconds, @@ -108,7 +117,6 @@ type localDNSCorefileData struct { type localDNSCorefileBlock struct { Zone string - IsRootDomain bool Listener string Upstream string NSID string @@ -190,7 +198,6 @@ func localDNSCorefileBlocks(overrides map[string]LocalDNSOverride, listener, def } blocks = append(blocks, localDNSCorefileBlock{ Zone: zone, - IsRootDomain: isRootDomain, Listener: listener, Upstream: upstream, NSID: nsid, diff --git a/pkg/config/internal/localdns/localdns_test.go b/pkg/config/internal/localdns/localdns_test.go index b6782c39..0752c98d 100644 --- a/pkg/config/internal/localdns/localdns_test.go +++ b/pkg/config/internal/localdns/localdns_test.go @@ -25,7 +25,7 @@ func TestCorefileTemplateGolden(t *testing.T) { }, KubeDNSOverrides: map[string]LocalDNSOverride{ ".": { - QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", + QueryLogging: "Error", Protocol: "PreferUDP", ForwardDestination: "ClusterCoreDNS", ForwardPolicy: "Random", MaxConcurrent: 1500, CacheDurationInSeconds: 90, ServeStaleDurationInSeconds: 120, ServeStale: "Verify", }, diff --git a/pkg/config/internal/localdns/testdata/localdns.corefile.golden b/pkg/config/internal/localdns/testdata/localdns.corefile.golden index 44b61cde..519896be 100644 --- a/pkg/config/internal/localdns/testdata/localdns.corefile.golden +++ b/pkg/config/internal/localdns/testdata/localdns.corefile.golden @@ -5,6 +5,9 @@ # - The metrics endpoint binds a runtime-resolved node address instead of :9253. # - The AgentBaker critical-FQDN hosts file is omitted because Unbounded does not # run the AKS hosts-file refresher. +# - Azure VM suffix guards for internal.cloudapp.net and reddog.microsoft.com +# are omitted because Flex Nodes may use non-Azure DNS where those names must +# be resolved according to the host's upstream policy. # Listener, upstream, cluster DNS, and metrics placeholders are resolved by # Unbounded after AKS Flex Node renders the profile-specific server blocks. health-check.localdns.local:53 { @@ -29,14 +32,6 @@ health-check.localdns.local:53 { loop nsid localdns prometheus {{ .MetricsAddress }} - template ANY ANY internal.cloudapp.net { - match "^(?:[^.]+\.){4,}internal\.cloudapp\.net\.$" - rcode NXDOMAIN - fallthrough - } - template ANY ANY reddog.microsoft.com { - rcode NXDOMAIN - } } cluster.local:53 { @@ -62,7 +57,6 @@ cluster.local:53 { errors bind {{ .ClusterListenerIP }} forward . {{ .ClusterDNSServiceIP }} { - force_tcp policy random max_concurrent 1500 } @@ -76,13 +70,5 @@ cluster.local:53 { loop nsid localdns-pod prometheus {{ .MetricsAddress }} - template ANY ANY internal.cloudapp.net { - match "^(?:[^.]+\.){4,}internal\.cloudapp\.net\.$" - rcode NXDOMAIN - fallthrough - } - template ANY ANY reddog.microsoft.com { - rcode NXDOMAIN - } } diff --git a/pkg/config/localdns_adapter_test.go b/pkg/config/localdns_adapter_test.go index 0be0025e..0afdea10 100644 --- a/pkg/config/localdns_adapter_test.go +++ b/pkg/config/localdns_adapter_test.go @@ -28,14 +28,11 @@ func TestToAgentConfigLocalDNS(t *testing.T) { if !strings.Contains(got.LocalDNS.CorefileTemplate, "{{ .ClusterDNSServiceIP }}") { t.Fatalf("CorefileTemplate missing cluster DNS placeholder:\n%s", got.LocalDNS.CorefileTemplate) } - for _, plugin := range []string{"nsid", "template"} { + for _, plugin := range []string{"log", "nsid"} { if !strings.Contains(strings.Join(got.LocalDNS.RequiredPlugins, ","), plugin) { t.Errorf("RequiredPlugins = %v, want %q", got.LocalDNS.RequiredPlugins, plugin) } } - if got.Kubelet.Labels["kubernetes.azure.com/localdns-state"] != "enabled" { - t.Fatalf("LocalDNS node label missing: %#v", got.Kubelet.Labels) - } if got.Cluster.ClusterDNS != "10.0.0.10" { t.Fatalf("ClusterDNS = %q, want original service IP", got.Cluster.ClusterDNS) } @@ -52,7 +49,4 @@ func TestToAgentConfigDisabledLocalDNS(t *testing.T) { if got.LocalDNS == nil || got.LocalDNS.Enabled { t.Fatalf("LocalDNS = %#v, want explicitly disabled", got.LocalDNS) } - if got.Kubelet.Labels["kubernetes.azure.com/localdns-state"] != "disabled" { - t.Fatalf("LocalDNS disabled node label missing: %#v", got.Kubelet.Labels) - } } diff --git a/pkg/config/localdns_test.go b/pkg/config/localdns_test.go index 81524ed1..cfa6d5fc 100644 --- a/pkg/config/localdns_test.go +++ b/pkg/config/localdns_test.go @@ -51,9 +51,9 @@ func TestLocalDNSProfileCorefileOptions(t *testing.T) { wantAbsent: []string{"force_tcp"}, }, { - name: "ClusterCoreDNS ForceTCP round robin verify", - override: LocalDNSOverride{QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", ForwardPolicy: "RoundRobin", ServeStale: "Verify"}, - want: []string{"errors", "forward . {{ .ClusterDNSServiceIP }}", "force_tcp", "policy round_robin", "serve_stale 3600s verify"}, + name: "ClusterCoreDNS ForceTCP round robin immediate", + override: LocalDNSOverride{QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", ForwardPolicy: "RoundRobin", ServeStale: "Immediate"}, + want: []string{"errors", "forward . {{ .ClusterDNSServiceIP }}", "force_tcp", "policy round_robin", "serve_stale 3600s immediate"}, }, { name: "random without stale", @@ -66,10 +66,14 @@ func TestLocalDNSProfileCorefileOptions(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() + vnetOverride := test.override + vnetOverride.ForwardDestination = "VnetDNS" + kubeOverride := test.override + kubeOverride.ForwardDestination = "ClusterCoreDNS" profile := &LocalDNSProfile{ Mode: LocalDNSModeRequired, - VnetDNSOverrides: map[string]LocalDNSOverride{".": test.override}, - KubeDNSOverrides: map[string]LocalDNSOverride{".": test.override}, + VnetDNSOverrides: map[string]LocalDNSOverride{".": vnetOverride}, + KubeDNSOverrides: map[string]LocalDNSOverride{".": kubeOverride}, } got, err := profile.CorefileTemplate() if err != nil { @@ -127,6 +131,9 @@ func TestLocalDNSProfileValidation(t *testing.T) { {name: "invalid policy", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ForwardPolicy: "First"}}}, wantErr: "forwardPolicy"}, {name: "invalid stale", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ServeStale: "Always"}}}, wantErr: "serveStale"}, {name: "negative cache", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {CacheDurationInSeconds: -1}}}, wantErr: "cacheDurationInSeconds"}, + {name: "VnetDNS root to cluster DNS", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}}}, wantErr: "must not be ClusterCoreDNS"}, + {name: "cluster zone to VnetDNS", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, KubeDNSOverrides: map[string]LocalDNSOverride{"cluster.local": {ForwardDestination: "VnetDNS"}}}, wantErr: "must not be VnetDNS"}, + {name: "ForceTCP with stale verification", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, KubeDNSOverrides: map[string]LocalDNSOverride{".": {Protocol: "ForceTCP", ServeStale: "Verify"}}}, wantErr: "requires protocol PreferUDP"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { From f2fb0abf7c00a4768ed069a86ece6487b387a523 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 21:23:08 +0000 Subject: [PATCH 14/17] fix: address LocalDNS review feedback --- hack/e2e/lib/upgrade-drift.sh | 2 +- hack/e2e/lib/validate.sh | 29 +++++++- pkg/config/internal/localdns/localdns.go | 73 ++++++++++++++++--- pkg/config/internal/localdns/localdns_test.go | 23 ++++++ pkg/config/localdns_test.go | 27 +++++++ 5 files changed, 139 insertions(+), 15 deletions(-) diff --git a/hack/e2e/lib/upgrade-drift.sh b/hack/e2e/lib/upgrade-drift.sh index 609e536d..8c283834 100644 --- a/hack/e2e/lib/upgrade-drift.sh +++ b/hack/e2e/lib/upgrade-drift.sh @@ -244,7 +244,7 @@ machine=$(machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}' test -n "${machine}" ! systemd-run --quiet --pipe --wait --machine="${machine}" systemctl cat localdns.service >/dev/null 2>&1 ! ip link show localdns >/dev/null 2>&1 -! iptables -w -t raw -S | grep -q 'unbounded-localdns: skip conntrack' +! nft list table ip unbounded_localdns >/dev/null 2>&1 systemd-run --quiet --pipe --wait --machine="${machine}" \ grep -q -- "--cluster-dns=${CLUSTER_DNS}" /etc/systemd/system/kubelet.service.d/20-node-config.conf REMOTE diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 92a32462..daf17a1e 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -6,6 +6,8 @@ # validate_node_joined - Wait for a specific node to appear in kubectl # validate_all_nodes - Verify MSI, token, offline, and kubeadm nodes joined # validate_npd_status - Verify node-problem-detector is active +# validate_localdns_status - Verify LocalDNS behavior +# validate_localdns_after_reboot - Verify LocalDNS after reboot # validate_node_absent - Wait for a node to disappear from kubectl # validate_all_nodes_absent - Verify all flex nodes are gone after unjoin # smoke_test