feat: support LocalDNS and generated kubelet configuration

docs/usages/configuration.md

@@ -122,6 +122,50 @@ At least one join or Azure authentication method must be configured. `azure.boot
 |------|------|-------------|--------------|
 | `networking.dnsServiceIP` | string | Cluster DNS service IP. | `10.0.0.10` |
 | `networking.cniVersion` | string | Optional CNI plugin version override. | `v1.6.2` |
+| `networking.localDNS` | object | Optional AKS LocalDNS profile using the same `mode`, `vnetDNSOverrides`, and `kubeDNSOverrides` shape accepted by `az aks nodepool --localdns-config`. | `{ "mode": "Required" }` |
+
+### AKS LocalDNS
+
+AKS Flex Node accepts the official AKS LocalDNS JSON profile under
+`networking.localDNS`. See [Configure LocalDNS in
+AKS](https://learn.microsoft.com/azure/aks/localdns-custom) for the supported
+fields and values. `Required` enables LocalDNS, `Disabled` disables it through
+repave, and `Preferred` validates the profile without enabling the service.
+
+```json
+{
+  "networking": {
+    "dnsServiceIP": "10.0.0.10",
+    "localDNS": {
+      "mode": "Required",
+      "vnetDNSOverrides": {
+        ".": {
+          "queryLogging": "Error",
+          "protocol": "PreferUDP",
+          "forwardDestination": "VnetDNS",
+          "forwardPolicy": "Sequential",
+          "maxConcurrent": 1000,
+          "cacheDurationInSeconds": 3600,
+          "serveStaleDurationInSeconds": 3600,
+          "serveStale": "Immediate"
+        }
+      },
+      "kubeDNSOverrides": {
+        ".": {
+          "queryLogging": "Error",
+          "protocol": "ForceTCP",
+          "forwardDestination": "ClusterCoreDNS",
+          "forwardPolicy": "Sequential",
+          "maxConcurrent": 1000,
+          "cacheDurationInSeconds": 3600,
+          "serveStaleDurationInSeconds": 3600,
+          "serveStale": "Immediate"
+        }
+      }
+    }
+  }
+}
+```
 
 ## Node
 
@@ -142,6 +186,12 @@ At least one join or Azure authentication method must be configured. `azure.boot
 | `node.kubelet.clusterFQDN` | string | Kubernetes API server FQDN. Required for bootstrap token mode. | `example.hcp.canadacentral.azmk8s.io` |
 | `node.kubelet.caCertData` | string | Base64-encoded cluster CA data. Required for bootstrap token mode. | `<base64-ca-data>` |
 | `node.kubelet.nodeIP` | string | Optional node IP override for kubelet `--node-ip`. | `10.0.0.4` |
+| `node.kubelet.imageCredentialProvider.configPath` | string | Optional absolute path inside the nspawn machine to a kubelet exec image credential provider configuration file or supported configuration directory. Must be set with `binDir`. | `/etc/kubernetes/credential-provider.yaml` |
+| `node.kubelet.imageCredentialProvider.binDir` | string | Optional absolute path inside the nspawn machine containing exec image credential provider binaries. Must be set with `configPath`. | `/usr/local/lib/kubelet-credential-providers` |
+
+Provider paths must be clean absolute machine paths without whitespace or systemd argument characters. Include the provider files in the OCI rootfs or expose them with read-only `bootstrap.additionalHostMounts`.
+
+The image credential provider executes a plugin to obtain short-lived pull credentials; it does not place registry passwords or tokens in the FlexNode configuration. Do not store static registry credentials in this file or provider configuration.
 
 ## Component Versions
 
@@ -348,3 +398,36 @@ Use `bootstrap.additionalHostMounts` to expose host files or directories inside
 ```
 
 Read-only entries render as systemd-nspawn `BindReadOnly=` directives; writable entries render as `Bind=` directives.
+
+### Image Pull Credential Provider
+
+Kubelet exec image credential providers can obtain short-lived registry credentials without storing tokens in the FlexNode config. The provider configuration and executable must exist inside the nspawn machine, either in the OCI rootfs or through host mounts:
+
+```json
+{
+  "bootstrap": {
+    "additionalHostMounts": [
+      {
+        "source": "/opt/aks-flex-node/credential-provider/config.yaml",
+        "target": "/etc/kubernetes/credential-provider.yaml",
+        "readOnly": true
+      },
+      {
+        "source": "/opt/aks-flex-node/credential-provider/bin",
+        "target": "/usr/local/lib/kubelet-credential-providers",
+        "readOnly": true
+      }
+    ]
+  },
+  "node": {
+    "kubelet": {
+      "imageCredentialProvider": {
+        "configPath": "/etc/kubernetes/credential-provider.yaml",
+        "binDir": "/usr/local/lib/kubelet-credential-providers"
+      }
+    }
+  }
+}
+```
+
+Provider binaries must be executable before the machine starts. Mount provider assets read-only unless the provider explicitly requires writable state.

go.mod

@@ -9,14 +9,14 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0
 	github.com/Azure/kubelogin v0.2.15
-	github.com/Azure/unbounded v0.2.2
+	github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806010226-43ec4d137b23
 	github.com/go-logr/logr v1.4.4
 	github.com/google/renameio/v2 v2.0.2
 	github.com/google/uuid v1.6.0
 	github.com/spf13/cobra v1.10.2
-	k8s.io/api v0.36.2
-	k8s.io/apimachinery v0.36.2
-	k8s.io/client-go v0.36.2
+	k8s.io/api v0.36.3
+	k8s.io/apimachinery v0.36.3
+	k8s.io/client-go v0.36.3
 	k8s.io/utils v0.0.0-20260319190234-28399d86e0b5
 	sigs.k8s.io/controller-runtime v0.24.1
 )
@@ -101,12 +101,12 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.36.0 // indirect
+	k8s.io/apiextensions-apiserver v0.36.3 // indirect
 	k8s.io/klog/v2 v2.140.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 // indirect
 	oras.land/oras-go/v2 v2.6.2 // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
 	sigs.k8s.io/randfill v1.0.0 // indirect
-	sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect
 )

go.sum

@@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys=
 github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls=
-github.com/Azure/unbounded v0.2.2 h1:uu5hYj20UBbrSAlf4qnDiMTGaR0xxsYnkdbTWTdBHX8=
-github.com/Azure/unbounded v0.2.2/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw=
+github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806010226-43ec4d137b23 h1:dhcPkHz7irmPQ8GU7IDhgZr8ag3Y1j3EeYS6znV4yug=
+github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806010226-43ec4d137b23/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
@@ -348,14 +348,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY=
-k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg=
-k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0=
-k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug=
-k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ=
-k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4=
-k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI=
-k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0=
+k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w=
+k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg=
+k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0=
+k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4=
+k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM=
+k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE=
+k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg=
+k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30=
 k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
 k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
 k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 h1:Sztf7ESG9tAXRW/ACJZjrj5jhdOUqS2KFRQT+CTvu78=
@@ -370,7 +370,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E
 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
 sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
 sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
-sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8=
-sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
 sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
 sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

hack/e2e/infra/main.bicep

@@ -236,6 +236,7 @@ output clusterFqdn string = aksCluster.properties.fqdn
 
 output msiVmName string = vmMsi.outputs.vmName
 output msiVmIp string = vmMsi.outputs.publicIpAddress
+output msiVmPrivateIp string = vmMsi.outputs.privateIpAddress
 output msiVmPrincipalId string = vmMsi.outputs.principalId
 
 output tokenVmName string = vmToken.outputs.vmName

hack/e2e/lib/infra.sh

@@ -107,14 +107,15 @@ infra_deploy() {
     --query properties.outputs \
     -o json)
 
-  local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_principal_id
+  local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_private_ip msi_vm_principal_id
   local token_vm_name token_vm_ip token_vm_private_ip offline_vm_name offline_vm_ip offline_vm_private_ip
   local kubeadm_vm_name kubeadm_vm_ip admin_username
 
   cluster_name=$(echo "${outputs}"    | jq -r '.clusterName.value')
   cluster_id=$(echo "${outputs}"      | jq -r '.clusterId.value')
   msi_vm_name=$(echo "${outputs}"     | jq -r '.msiVmName.value')
   msi_vm_ip=$(echo "${outputs}"       | jq -r '.msiVmIp.value')
+  msi_vm_private_ip=$(echo "${outputs}" | jq -r '.msiVmPrivateIp.value // ""')
   msi_vm_principal_id=$(echo "${outputs}" | jq -r '.msiVmPrincipalId.value')
   token_vm_name=$(echo "${outputs}"   | jq -r '.tokenVmName.value')
   token_vm_ip=$(echo "${outputs}"     | jq -r '.tokenVmIp.value')
@@ -126,6 +127,10 @@ infra_deploy() {
   kubeadm_vm_ip=$(echo "${outputs}"   | jq -r '.kubeadmVmIp.value')
   admin_username=$(echo "${outputs}"  | jq -r '.adminUsername.value')
 
+  if [[ -z "${msi_vm_private_ip}" ]] || ! is_valid_ipv4 "${msi_vm_private_ip}"; then
+    log_error "Missing or invalid MSI VM private IP from deployment outputs: '${msi_vm_private_ip}'"
+    return 1
+  fi
   if [[ -z "${token_vm_private_ip}" ]] || ! is_valid_ipv4 "${token_vm_private_ip}"; then
     log_error "Missing or invalid token VM private IP from deployment outputs: '${token_vm_private_ip}'"
     return 1
@@ -140,6 +145,7 @@ infra_deploy() {
   state_set "cluster_id"           "${cluster_id}"
   state_set "msi_vm_name"          "${msi_vm_name}"
   state_set "msi_vm_ip"            "${msi_vm_ip}"
+  state_set "msi_vm_private_ip"    "${msi_vm_private_ip}"
   state_set "msi_vm_principal_id"  "${msi_vm_principal_id}"
   state_set "token_vm_name"        "${token_vm_name}"
   state_set "token_vm_ip"          "${token_vm_ip}"

hack/e2e/lib/node-join-msi.sh

@@ -14,6 +14,63 @@ readonly _E2E_NODE_JOIN_MSI_LOADED=1
 # shellcheck disable=SC1091
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
+# ---------------------------------------------------------------------------
+# prepare_localdns_host_resolver - Remove DHCP search/routing domains.
+#
+# Unbounded intentionally rejects systemd-resolved split-DNS layouts because
+# flattening per-domain routing into one LocalDNS upstream list changes DNS
+# semantics. Azure's Ubuntu image supplies a DHCP search domain by default, so
+# put the disposable E2E host into the supported single-upstream layout before
+# LocalDNS preflight runs.
+# ---------------------------------------------------------------------------
+prepare_localdns_host_resolver() {
+  local vm_ip="$1"
+
+  log_info "Configuring the MSI host resolver for LocalDNS..."
+  remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE'
+set -euo pipefail
+
+interface="$(ip -4 route show default | awk 'NR == 1 { print $5 }')"
+if [[ ! "${interface}" =~ ^[a-zA-Z0-9_.:-]+$ ]]; then
+  echo "could not determine a safe primary interface name: ${interface}" >&2
+  exit 1
+fi
+
+cat > /etc/netplan/99-aks-flex-localdns.yaml <<EOF
+network:
+  version: 2
+  ethernets:
+    ${interface}:
+      dhcp4-overrides:
+        use-domains: false
+EOF
+chmod 0600 /etc/netplan/99-aks-flex-localdns.yaml
+netplan generate
+netplan apply
+
+# Match Unbounded's supported-domain rule: no domain is allowed except the
+# catch-all routing domain (~.).
+if resolvectl domain | awk -F: '
+  {
+    for (i = 2; i <= NF; i++) {
+      count = split($i, domains, /[[:space:]]+/)
+      for (j = 1; j <= count; j++)
+        if (domains[j] != "" && domains[j] != "~.")
+          found = 1
+    }
+  }
+  END { exit !found }
+'; then
+  echo "unsupported systemd-resolved domain remains after netplan apply:" >&2
+  resolvectl domain >&2
+  exit 1
+fi
+
+resolvectl domain
+resolvectl dns "${interface}"
+REMOTE
+}
+
 # ---------------------------------------------------------------------------
 # node_join_msi - Join the MSI VM
 # ---------------------------------------------------------------------------
@@ -24,6 +81,8 @@ node_join_msi() {
 
   local vm_ip
   vm_ip="$(state_get msi_vm_ip)"
+  local vm_private_ip
+  vm_private_ip="$(state_get msi_vm_private_ip)"
   local cluster_id
   cluster_id="$(state_get cluster_id)"
   local subscription_id
@@ -55,7 +114,8 @@ node_join_msi() {
   "node": {
     "kubelet": {
       "clusterFQDN": "${server_url}",
-      "caCertData": "${ca_cert_data}"
+      "caCertData": "${ca_cert_data}",
+      "nodeIP": "${vm_private_ip}"
     }
   },
   "agent": {
@@ -67,6 +127,35 @@ node_join_msi() {
     },
     "requireMachineRegistration": true
   },
+  "networking": {
+    "localDNS": {
+      "mode": "Required",
+      "vnetDNSOverrides": {
+        ".": {
+          "queryLogging": "Error",
+          "protocol": "PreferUDP",
+          "forwardDestination": "VnetDNS",
+          "forwardPolicy": "Sequential",
+          "maxConcurrent": 1000,
+          "cacheDurationInSeconds": 3600,
+          "serveStaleDurationInSeconds": 3600,
+          "serveStale": "Immediate"
+        }
+      },
+      "kubeDNSOverrides": {
+        ".": {
+          "queryLogging": "Error",
+          "protocol": "ForceTCP",
+          "forwardDestination": "ClusterCoreDNS",
+          "forwardPolicy": "Sequential",
+          "maxConcurrent": 1000,
+          "cacheDurationInSeconds": 3600,
+          "serveStaleDurationInSeconds": 3600,
+          "serveStale": "Immediate"
+        }
+      }
+    }
+  },
   "components": {
     "kubernetes": "${E2E_KUBERNETES_VERSION}",
     "containerd": "${E2E_CONTAINERD_VERSION}",
@@ -75,7 +164,10 @@ node_join_msi() {
 }
 EOF
 
-  # Step 2: Publish the AKS Machine goal and deploy the agent.
+  # Step 2: Put systemd-resolved into the layout supported by LocalDNS.
+  prepare_localdns_host_resolver "${vm_ip}"
+
+  # Step 3: Publish the AKS Machine goal and deploy the agent.
   ensure_flex_controller
   machine_configmap_upsert "$(state_get msi_vm_name)" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}"
   _deploy_and_start_agent "${vm_ip}" "${config_file}" "aks-flex-node-msi"