diff --git a/docs/usages/configuration.md b/docs/usages/configuration.md index ccd992aa..773e7190 100644 --- a/docs/usages/configuration.md +++ b/docs/usages/configuration.md @@ -122,6 +122,50 @@ At least one join or Azure authentication method must be configured. `azure.boot |------|------|-------------|--------------| | `networking.dnsServiceIP` | string | Cluster DNS service IP. | `10.0.0.10` | | `networking.cniVersion` | string | Optional CNI plugin version override. | `v1.6.2` | +| `networking.localDNS` | object | Optional AKS LocalDNS profile using the same `mode`, `vnetDNSOverrides`, and `kubeDNSOverrides` shape accepted by `az aks nodepool --localdns-config`. | `{ "mode": "Required" }` | + +### AKS LocalDNS + +AKS Flex Node accepts the official AKS LocalDNS JSON profile under +`networking.localDNS`. See [Configure LocalDNS in +AKS](https://learn.microsoft.com/azure/aks/localdns-custom) for the supported +fields and values. `Required` enables LocalDNS, `Disabled` disables it through +repave, and `Preferred` validates the profile without enabling the service. + +```json +{ + "networking": { + "dnsServiceIP": "10.0.0.10", + "localDNS": { + "mode": "Required", + "vnetDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "ForceTCP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } +} +``` ## Node diff --git a/hack/e2e/infra/main.bicep b/hack/e2e/infra/main.bicep index a07e5984..ec44699d 100644 --- a/hack/e2e/infra/main.bicep +++ b/hack/e2e/infra/main.bicep @@ -236,6 +236,7 @@ output clusterFqdn string = aksCluster.properties.fqdn output msiVmName string = vmMsi.outputs.vmName output msiVmIp string = vmMsi.outputs.publicIpAddress +output msiVmPrivateIp string = vmMsi.outputs.privateIpAddress output msiVmPrincipalId string = vmMsi.outputs.principalId output tokenVmName string = vmToken.outputs.vmName diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index cc3d457b..d7ba0224 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -107,7 +107,7 @@ infra_deploy() { --query properties.outputs \ -o json) - local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_principal_id + local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_private_ip msi_vm_principal_id local token_vm_name token_vm_ip token_vm_private_ip offline_vm_name offline_vm_ip offline_vm_private_ip local kubeadm_vm_name kubeadm_vm_ip admin_username @@ -115,6 +115,7 @@ infra_deploy() { cluster_id=$(echo "${outputs}" | jq -r '.clusterId.value') msi_vm_name=$(echo "${outputs}" | jq -r '.msiVmName.value') msi_vm_ip=$(echo "${outputs}" | jq -r '.msiVmIp.value') + msi_vm_private_ip=$(echo "${outputs}" | jq -r '.msiVmPrivateIp.value // ""') msi_vm_principal_id=$(echo "${outputs}" | jq -r '.msiVmPrincipalId.value') token_vm_name=$(echo "${outputs}" | jq -r '.tokenVmName.value') token_vm_ip=$(echo "${outputs}" | jq -r '.tokenVmIp.value') @@ -126,6 +127,10 @@ infra_deploy() { kubeadm_vm_ip=$(echo "${outputs}" | jq -r '.kubeadmVmIp.value') admin_username=$(echo "${outputs}" | jq -r '.adminUsername.value') + if [[ -z "${msi_vm_private_ip}" ]] || ! is_valid_ipv4 "${msi_vm_private_ip}"; then + log_error "Missing or invalid MSI VM private IP from deployment outputs: '${msi_vm_private_ip}'" + return 1 + fi if [[ -z "${token_vm_private_ip}" ]] || ! is_valid_ipv4 "${token_vm_private_ip}"; then log_error "Missing or invalid token VM private IP from deployment outputs: '${token_vm_private_ip}'" return 1 @@ -140,6 +145,7 @@ infra_deploy() { state_set "cluster_id" "${cluster_id}" state_set "msi_vm_name" "${msi_vm_name}" state_set "msi_vm_ip" "${msi_vm_ip}" + state_set "msi_vm_private_ip" "${msi_vm_private_ip}" state_set "msi_vm_principal_id" "${msi_vm_principal_id}" state_set "token_vm_name" "${token_vm_name}" state_set "token_vm_ip" "${token_vm_ip}" diff --git a/hack/e2e/lib/node-join-msi.sh b/hack/e2e/lib/node-join-msi.sh index e9167574..6a94715e 100644 --- a/hack/e2e/lib/node-join-msi.sh +++ b/hack/e2e/lib/node-join-msi.sh @@ -14,6 +14,63 @@ readonly _E2E_NODE_JOIN_MSI_LOADED=1 # shellcheck disable=SC1091 source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +# --------------------------------------------------------------------------- +# prepare_localdns_host_resolver - Remove DHCP search/routing domains. +# +# Unbounded intentionally rejects systemd-resolved split-DNS layouts because +# flattening per-domain routing into one LocalDNS upstream list changes DNS +# semantics. Azure's Ubuntu image supplies a DHCP search domain by default, so +# put the disposable E2E host into the supported single-upstream layout before +# LocalDNS preflight runs. +# --------------------------------------------------------------------------- +prepare_localdns_host_resolver() { + local vm_ip="$1" + + log_info "Configuring the MSI host resolver for LocalDNS..." + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail + +interface="$(ip -4 route show default | awk 'NR == 1 { print $5 }')" +if [[ ! "${interface}" =~ ^[a-zA-Z0-9_.:-]+$ ]]; then + echo "could not determine a safe primary interface name: ${interface}" >&2 + exit 1 +fi + +cat > /etc/netplan/99-aks-flex-localdns.yaml <&2 + resolvectl domain >&2 + exit 1 +fi + +resolvectl domain +resolvectl dns "${interface}" +REMOTE +} + # --------------------------------------------------------------------------- # node_join_msi - Join the MSI VM # --------------------------------------------------------------------------- @@ -24,6 +81,8 @@ node_join_msi() { local vm_ip vm_ip="$(state_get msi_vm_ip)" + local vm_private_ip + vm_private_ip="$(state_get msi_vm_private_ip)" local cluster_id cluster_id="$(state_get cluster_id)" local subscription_id @@ -55,7 +114,8 @@ node_join_msi() { "node": { "kubelet": { "clusterFQDN": "${server_url}", - "caCertData": "${ca_cert_data}" + "caCertData": "${ca_cert_data}", + "nodeIP": "${vm_private_ip}" } }, "agent": { @@ -67,6 +127,35 @@ node_join_msi() { }, "requireMachineRegistration": true }, + "networking": { + "localDNS": { + "mode": "Required", + "vnetDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "ForceTCP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + }, "components": { "kubernetes": "${E2E_KUBERNETES_VERSION}", "containerd": "${E2E_CONTAINERD_VERSION}", @@ -75,7 +164,10 @@ node_join_msi() { } EOF - # Step 2: Publish the AKS Machine goal and deploy the agent. + # Step 2: Put systemd-resolved into the layout supported by LocalDNS. + prepare_localdns_host_resolver "${vm_ip}" + + # Step 3: Publish the AKS Machine goal and deploy the agent. ensure_flex_controller machine_configmap_upsert "$(state_get msi_vm_name)" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}" _deploy_and_start_agent "${vm_ip}" "${config_file}" "aks-flex-node-msi" diff --git a/hack/e2e/lib/upgrade-drift.sh b/hack/e2e/lib/upgrade-drift.sh index f1be588f..24f9478e 100644 --- a/hack/e2e/lib/upgrade-drift.sh +++ b/hack/e2e/lib/upgrade-drift.sh @@ -209,11 +209,69 @@ upgrade_drift_mode() { log_success "${mode} controller machine repave passed in $(timer_elapsed "${start}")s" } +localdns_disable_repave_msi() { + log_section "LocalDNS Disable Repave (MSI node)" + local desired_version settings_version vm_name vm_ip snapshot old_active_machine old_state old_version old_settings_version old_node_uid + desired_version="$(_cluster_current_kubernetes_version)" + settings_version="localdns-disabled-$(date +%s)" + vm_name="$(_mode_vm_name msi)" + vm_ip="$(_mode_vm_ip msi)" + + snapshot="$(_remote_active_machine_snapshot "${vm_ip}")" + IFS='|' read -r old_active_machine old_state old_version old_settings_version <<<"${snapshot}" + old_node_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" + + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail +config=/etc/aks-flex-node/config.json +tmp=$(mktemp /etc/aks-flex-node/config.json.XXXXXX) +jq '.networking.localDNS.mode = "Disabled"' "${config}" > "${tmp}" +chmod 0600 "${tmp}" +mv "${tmp}" "${config}" +systemctl restart aks-flex-node-agent.service +systemctl is-active --quiet aks-flex-node-agent.service +REMOTE + + _trigger_mode_repave msi "${desired_version}" "${settings_version}" + _wait_for_mode_repave msi "${desired_version}" "${settings_version}" "${old_active_machine}" "${old_node_uid}" + + local cluster_dns + cluster_dns="$(kubectl -n kube-system get service kube-dns -o jsonpath='{.spec.clusterIP}')" + + remote_exec "${vm_ip}" "CLUSTER_DNS=${cluster_dns} sudo -E bash -s" <<'REMOTE' +set -euo pipefail +machine=$(machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}') +test -n "${machine}" +if systemd-run --quiet --pipe --wait --machine="${machine}" systemctl cat localdns.service >/dev/null 2>&1; then + echo "localdns.service still exists after disabling LocalDNS" >&2 + exit 1 +fi +if ip link show localdns >/dev/null 2>&1; then + echo "localdns interface still exists after disabling LocalDNS" >&2 + exit 1 +fi +if nft list table ip unbounded_localdns >/dev/null 2>&1; then + echo "unbounded_localdns nftables table still exists after disabling LocalDNS" >&2 + exit 1 +fi +systemd-run --quiet --pipe --wait --machine="${machine}" \ + awk -v expected="${CLUSTER_DNS}" ' + $1 == "clusterDNS:" { in_cluster_dns = 1; next } + in_cluster_dns && $1 == "-" && $2 == expected { found = 1 } + in_cluster_dns && $1 != "-" { in_cluster_dns = 0 } + END { exit !found } + ' /var/lib/kubelet/config.yaml +REMOTE + + log_success "MSI LocalDNS disable-through-repave validation passed" +} + upgrade_drift_all() { log_section "Controller Machine Repave (all modes)" upgrade_drift_mode msi upgrade_drift_mode token upgrade_drift_mode kubeadm + localdns_disable_repave_msi } upgrade_drift_msi() { upgrade_drift_mode msi; } diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 02ab9f6c..0857bbfc 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -6,6 +6,8 @@ # validate_node_joined - Wait for a specific node to appear in kubectl # validate_all_nodes - Verify MSI, token, offline, and kubeadm nodes joined # validate_npd_status - Verify node-problem-detector is active +# validate_localdns_status - Verify LocalDNS behavior +# validate_localdns_after_reboot - Verify LocalDNS after nspawn reboot # validate_node_absent - Wait for a node to disappear from kubectl # validate_all_nodes_absent - Verify all flex nodes are gone after unjoin # smoke_test