feat: support AKS LocalDNS profiles

docs/usages/configuration.md

@@ -122,6 +122,50 @@ At least one join or Azure authentication method must be configured. `azure.boot
 |------|------|-------------|--------------|
 | `networking.dnsServiceIP` | string | Cluster DNS service IP. | `10.0.0.10` |
 | `networking.cniVersion` | string | Optional CNI plugin version override. | `v1.6.2` |
+| `networking.localDNS` | object | Optional AKS LocalDNS profile using the same `mode`, `vnetDNSOverrides`, and `kubeDNSOverrides` shape accepted by `az aks nodepool --localdns-config`. | `{ "mode": "Required" }` |
+
+### AKS LocalDNS
+
+AKS Flex Node accepts the official AKS LocalDNS JSON profile under
+`networking.localDNS`. See [Configure LocalDNS in
+AKS](https://learn.microsoft.com/azure/aks/localdns-custom) for the supported
+fields and values. `Required` enables LocalDNS, `Disabled` disables it through
+repave, and `Preferred` validates the profile without enabling the service.
+
+```json
+{
+  "networking": {
+    "dnsServiceIP": "10.0.0.10",
+    "localDNS": {
+      "mode": "Required",
+      "vnetDNSOverrides": {
+        ".": {
+          "queryLogging": "Error",
+          "protocol": "PreferUDP",
+          "forwardDestination": "VnetDNS",
+          "forwardPolicy": "Sequential",
+          "maxConcurrent": 1000,
+          "cacheDurationInSeconds": 3600,
+          "serveStaleDurationInSeconds": 3600,
+          "serveStale": "Immediate"
+        }
+      },
+      "kubeDNSOverrides": {
+        ".": {
+          "queryLogging": "Error",
+          "protocol": "ForceTCP",
+          "forwardDestination": "ClusterCoreDNS",
+          "forwardPolicy": "Sequential",
+          "maxConcurrent": 1000,
+          "cacheDurationInSeconds": 3600,
+          "serveStaleDurationInSeconds": 3600,
+          "serveStale": "Immediate"
+        }
+      }
+    }
+  }
+}
+```
 
 ## Node
 

hack/e2e/infra/main.bicep

@@ -236,6 +236,7 @@ output clusterFqdn string = aksCluster.properties.fqdn
 
 output msiVmName string = vmMsi.outputs.vmName
 output msiVmIp string = vmMsi.outputs.publicIpAddress
+output msiVmPrivateIp string = vmMsi.outputs.privateIpAddress
 output msiVmPrincipalId string = vmMsi.outputs.principalId
 
 output tokenVmName string = vmToken.outputs.vmName

hack/e2e/lib/infra.sh

@@ -107,14 +107,15 @@ infra_deploy() {
     --query properties.outputs \
     -o json)
 
-  local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_principal_id
+  local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_private_ip msi_vm_principal_id
   local token_vm_name token_vm_ip token_vm_private_ip offline_vm_name offline_vm_ip offline_vm_private_ip
   local kubeadm_vm_name kubeadm_vm_ip admin_username
 
   cluster_name=$(echo "${outputs}"    | jq -r '.clusterName.value')
   cluster_id=$(echo "${outputs}"      | jq -r '.clusterId.value')
   msi_vm_name=$(echo "${outputs}"     | jq -r '.msiVmName.value')
   msi_vm_ip=$(echo "${outputs}"       | jq -r '.msiVmIp.value')
+  msi_vm_private_ip=$(echo "${outputs}" | jq -r '.msiVmPrivateIp.value // ""')
   msi_vm_principal_id=$(echo "${outputs}" | jq -r '.msiVmPrincipalId.value')
   token_vm_name=$(echo "${outputs}"   | jq -r '.tokenVmName.value')
   token_vm_ip=$(echo "${outputs}"     | jq -r '.tokenVmIp.value')
@@ -126,6 +127,10 @@ infra_deploy() {
   kubeadm_vm_ip=$(echo "${outputs}"   | jq -r '.kubeadmVmIp.value')
   admin_username=$(echo "${outputs}"  | jq -r '.adminUsername.value')
 
+  if [[ -z "${msi_vm_private_ip}" ]] || ! is_valid_ipv4 "${msi_vm_private_ip}"; then
+    log_error "Missing or invalid MSI VM private IP from deployment outputs: '${msi_vm_private_ip}'"
+    return 1
+  fi
   if [[ -z "${token_vm_private_ip}" ]] || ! is_valid_ipv4 "${token_vm_private_ip}"; then
     log_error "Missing or invalid token VM private IP from deployment outputs: '${token_vm_private_ip}'"
     return 1
@@ -140,6 +145,7 @@ infra_deploy() {
   state_set "cluster_id"           "${cluster_id}"
   state_set "msi_vm_name"          "${msi_vm_name}"
   state_set "msi_vm_ip"            "${msi_vm_ip}"
+  state_set "msi_vm_private_ip"    "${msi_vm_private_ip}"
   state_set "msi_vm_principal_id"  "${msi_vm_principal_id}"
   state_set "token_vm_name"        "${token_vm_name}"
   state_set "token_vm_ip"          "${token_vm_ip}"

hack/e2e/lib/node-join-msi.sh

@@ -14,6 +14,63 @@ readonly _E2E_NODE_JOIN_MSI_LOADED=1
 # shellcheck disable=SC1091
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
+# ---------------------------------------------------------------------------
+# prepare_localdns_host_resolver - Remove DHCP search/routing domains.
+#
+# Unbounded intentionally rejects systemd-resolved split-DNS layouts because
+# flattening per-domain routing into one LocalDNS upstream list changes DNS
+# semantics. Azure's Ubuntu image supplies a DHCP search domain by default, so
+# put the disposable E2E host into the supported single-upstream layout before
+# LocalDNS preflight runs.
+# ---------------------------------------------------------------------------
+prepare_localdns_host_resolver() {
+  local vm_ip="$1"
+
+  log_info "Configuring the MSI host resolver for LocalDNS..."
+  remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE'
+set -euo pipefail
+
+interface="$(ip -4 route show default | awk 'NR == 1 { print $5 }')"
+if [[ ! "${interface}" =~ ^[a-zA-Z0-9_.:-]+$ ]]; then
+  echo "could not determine a safe primary interface name: ${interface}" >&2
+  exit 1
+fi
+
+cat > /etc/netplan/99-aks-flex-localdns.yaml <<EOF
+network:
+  version: 2
+  ethernets:
+    ${interface}:
+      dhcp4-overrides:
+        use-domains: false
+EOF
+chmod 0600 /etc/netplan/99-aks-flex-localdns.yaml
+netplan generate
+netplan apply
+
+# Match Unbounded's supported-domain rule: no domain is allowed except the
+# catch-all routing domain (~.).
+if resolvectl domain | awk -F: '
+  {
+    for (i = 2; i <= NF; i++) {
+      count = split($i, domains, /[[:space:]]+/)
+      for (j = 1; j <= count; j++)
+        if (domains[j] != "" && domains[j] != "~.")
+          found = 1
+    }
+  }
+  END { exit !found }
+'; then
+  echo "unsupported systemd-resolved domain remains after netplan apply:" >&2
+  resolvectl domain >&2
+  exit 1
+fi
+
+resolvectl domain
+resolvectl dns "${interface}"
+REMOTE
+}
+
 # ---------------------------------------------------------------------------
 # node_join_msi - Join the MSI VM
 # ---------------------------------------------------------------------------
@@ -24,6 +81,8 @@ node_join_msi() {
 
   local vm_ip
   vm_ip="$(state_get msi_vm_ip)"
+  local vm_private_ip
+  vm_private_ip="$(state_get msi_vm_private_ip)"
   local cluster_id
   cluster_id="$(state_get cluster_id)"
   local subscription_id
@@ -55,7 +114,8 @@ node_join_msi() {
   "node": {
     "kubelet": {
       "clusterFQDN": "${server_url}",
-      "caCertData": "${ca_cert_data}"
+      "caCertData": "${ca_cert_data}",
+      "nodeIP": "${vm_private_ip}"
     }
   },
   "agent": {
@@ -67,6 +127,35 @@ node_join_msi() {
     },
     "requireMachineRegistration": true
   },
+  "networking": {
+    "localDNS": {
+      "mode": "Required",
+      "vnetDNSOverrides": {
+        ".": {
+          "queryLogging": "Error",
+          "protocol": "PreferUDP",
+          "forwardDestination": "VnetDNS",
+          "forwardPolicy": "Sequential",
+          "maxConcurrent": 1000,
+          "cacheDurationInSeconds": 3600,
+          "serveStaleDurationInSeconds": 3600,
+          "serveStale": "Immediate"
+        }
+      },
+      "kubeDNSOverrides": {
+        ".": {
+          "queryLogging": "Error",
+          "protocol": "ForceTCP",
+          "forwardDestination": "ClusterCoreDNS",
+          "forwardPolicy": "Sequential",
+          "maxConcurrent": 1000,
+          "cacheDurationInSeconds": 3600,
+          "serveStaleDurationInSeconds": 3600,
+          "serveStale": "Immediate"
+        }
+      }
+    }
+  },
   "components": {
     "kubernetes": "${E2E_KUBERNETES_VERSION}",
     "containerd": "${E2E_CONTAINERD_VERSION}",
@@ -75,7 +164,10 @@ node_join_msi() {
 }
 EOF
 
-  # Step 2: Publish the AKS Machine goal and deploy the agent.
+  # Step 2: Put systemd-resolved into the layout supported by LocalDNS.
+  prepare_localdns_host_resolver "${vm_ip}"
+
+  # Step 3: Publish the AKS Machine goal and deploy the agent.
   ensure_flex_controller
   machine_configmap_upsert "$(state_get msi_vm_name)" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}"
   _deploy_and_start_agent "${vm_ip}" "${config_file}" "aks-flex-node-msi"

hack/e2e/lib/upgrade-drift.sh

@@ -209,11 +209,69 @@ upgrade_drift_mode() {
   log_success "${mode} controller machine repave passed in $(timer_elapsed "${start}")s"
 }
 
+localdns_disable_repave_msi() {
+  log_section "LocalDNS Disable Repave (MSI node)"
+  local desired_version settings_version vm_name vm_ip snapshot old_active_machine old_state old_version old_settings_version old_node_uid
+  desired_version="$(_cluster_current_kubernetes_version)"
+  settings_version="localdns-disabled-$(date +%s)"
+  vm_name="$(_mode_vm_name msi)"
+  vm_ip="$(_mode_vm_ip msi)"
+
+  snapshot="$(_remote_active_machine_snapshot "${vm_ip}")"
+  IFS='|' read -r old_active_machine old_state old_version old_settings_version <<<"${snapshot}"
+  old_node_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')"
+
+  remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE'
+set -euo pipefail
+config=/etc/aks-flex-node/config.json
+tmp=$(mktemp /etc/aks-flex-node/config.json.XXXXXX)
+jq '.networking.localDNS.mode = "Disabled"' "${config}" > "${tmp}"
+chmod 0600 "${tmp}"
+mv "${tmp}" "${config}"
+systemctl restart aks-flex-node-agent.service
+systemctl is-active --quiet aks-flex-node-agent.service
+REMOTE
+
+  _trigger_mode_repave msi "${desired_version}" "${settings_version}"
+  _wait_for_mode_repave msi "${desired_version}" "${settings_version}" "${old_active_machine}" "${old_node_uid}"
+
+  local cluster_dns
+  cluster_dns="$(kubectl -n kube-system get service kube-dns -o jsonpath='{.spec.clusterIP}')"
+
+  remote_exec "${vm_ip}" "CLUSTER_DNS=${cluster_dns} sudo -E bash -s" <<'REMOTE'
+set -euo pipefail
+machine=$(machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}')
+test -n "${machine}"
+if systemd-run --quiet --pipe --wait --machine="${machine}" systemctl cat localdns.service >/dev/null 2>&1; then
+  echo "localdns.service still exists after disabling LocalDNS" >&2
+  exit 1
+fi
+if ip link show localdns >/dev/null 2>&1; then
+  echo "localdns interface still exists after disabling LocalDNS" >&2
+  exit 1
+fi
+if nft list table ip unbounded_localdns >/dev/null 2>&1; then
+  echo "unbounded_localdns nftables table still exists after disabling LocalDNS" >&2
+  exit 1
+fi
+systemd-run --quiet --pipe --wait --machine="${machine}" \
+  awk -v expected="${CLUSTER_DNS}" '
+    $1 == "clusterDNS:" { in_cluster_dns = 1; next }
+    in_cluster_dns && $1 == "-" && $2 == expected { found = 1 }
+    in_cluster_dns && $1 != "-" { in_cluster_dns = 0 }
+    END { exit !found }
+  ' /var/lib/kubelet/config.yaml
+REMOTE
+
+  log_success "MSI LocalDNS disable-through-repave validation passed"
+}
+
 upgrade_drift_all() {
   log_section "Controller Machine Repave (all modes)"
   upgrade_drift_mode msi
   upgrade_drift_mode token
   upgrade_drift_mode kubeadm
+  localdns_disable_repave_msi
 }
 
 upgrade_drift_msi() { upgrade_drift_mode msi; }