diff --git a/docs/usages/configuration.md b/docs/usages/configuration.md index 11d8084e..773e7190 100644 --- a/docs/usages/configuration.md +++ b/docs/usages/configuration.md @@ -122,6 +122,50 @@ At least one join or Azure authentication method must be configured. `azure.boot |------|------|-------------|--------------| | `networking.dnsServiceIP` | string | Cluster DNS service IP. | `10.0.0.10` | | `networking.cniVersion` | string | Optional CNI plugin version override. | `v1.6.2` | +| `networking.localDNS` | object | Optional AKS LocalDNS profile using the same `mode`, `vnetDNSOverrides`, and `kubeDNSOverrides` shape accepted by `az aks nodepool --localdns-config`. | `{ "mode": "Required" }` | + +### AKS LocalDNS + +AKS Flex Node accepts the official AKS LocalDNS JSON profile under +`networking.localDNS`. See [Configure LocalDNS in +AKS](https://learn.microsoft.com/azure/aks/localdns-custom) for the supported +fields and values. `Required` enables LocalDNS, `Disabled` disables it through +repave, and `Preferred` validates the profile without enabling the service. + +```json +{ + "networking": { + "dnsServiceIP": "10.0.0.10", + "localDNS": { + "mode": "Required", + "vnetDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "ForceTCP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + } +} +``` ## Node @@ -142,6 +186,12 @@ At least one join or Azure authentication method must be configured. `azure.boot | `node.kubelet.clusterFQDN` | string | Kubernetes API server FQDN. Required for bootstrap token mode. | `example.hcp.canadacentral.azmk8s.io` | | `node.kubelet.caCertData` | string | Base64-encoded cluster CA data. Required for bootstrap token mode. | `` | | `node.kubelet.nodeIP` | string | Optional node IP override for kubelet `--node-ip`. | `10.0.0.4` | +| `node.kubelet.imageCredentialProvider.configPath` | string | Optional absolute path inside the nspawn machine to a kubelet exec image credential provider configuration file or supported configuration directory. Must be set with `binDir`. | `/etc/kubernetes/credential-provider.yaml` | +| `node.kubelet.imageCredentialProvider.binDir` | string | Optional absolute path inside the nspawn machine containing exec image credential provider binaries. Must be set with `configPath`. | `/usr/local/lib/kubelet-credential-providers` | + +Provider paths must be clean absolute machine paths without whitespace or systemd argument characters. Include the provider files in the OCI rootfs or expose them with read-only `bootstrap.additionalHostMounts`. + +The image credential provider executes a plugin to obtain short-lived pull credentials; it does not place registry passwords or tokens in the FlexNode configuration. Do not store static registry credentials in this file or provider configuration. ## Component Versions @@ -348,3 +398,36 @@ Use `bootstrap.additionalHostMounts` to expose host files or directories inside ``` Read-only entries render as systemd-nspawn `BindReadOnly=` directives; writable entries render as `Bind=` directives. + +### Image Pull Credential Provider + +Kubelet exec image credential providers can obtain short-lived registry credentials without storing tokens in the FlexNode config. The provider configuration and executable must exist inside the nspawn machine, either in the OCI rootfs or through host mounts: + +```json +{ + "bootstrap": { + "additionalHostMounts": [ + { + "source": "/opt/aks-flex-node/credential-provider/config.yaml", + "target": "/etc/kubernetes/credential-provider.yaml", + "readOnly": true + }, + { + "source": "/opt/aks-flex-node/credential-provider/bin", + "target": "/usr/local/lib/kubelet-credential-providers", + "readOnly": true + } + ] + }, + "node": { + "kubelet": { + "imageCredentialProvider": { + "configPath": "/etc/kubernetes/credential-provider.yaml", + "binDir": "/usr/local/lib/kubelet-credential-providers" + } + } + } +} +``` + +Provider binaries must be executable before the machine starts. Mount provider assets read-only unless the provider explicitly requires writable state. diff --git a/go.mod b/go.mod index f7009d23..a69c755e 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,14 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.2 + github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806010226-43ec4d137b23 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.2 - k8s.io/api v0.36.2 - k8s.io/apimachinery v0.36.2 - k8s.io/client-go v0.36.2 + k8s.io/api v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/client-go v0.36.3 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 ) @@ -101,12 +101,12 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.36.0 // indirect + k8s.io/apiextensions-apiserver v0.36.3 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 // indirect oras.land/oras-go/v2 v2.6.2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index dd35efd4..cb88b8ae 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.2 h1:uu5hYj20UBbrSAlf4qnDiMTGaR0xxsYnkdbTWTdBHX8= -github.com/Azure/unbounded v0.2.2/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw= +github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806010226-43ec4d137b23 h1:dhcPkHz7irmPQ8GU7IDhgZr8ag3Y1j3EeYS6znV4yug= +github.com/Azure/unbounded v0.2.3-alpha.1.0.20260806010226-43ec4d137b23/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= @@ -348,14 +348,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= -k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= -k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= -k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= -k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= -k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= -k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= -k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= +k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0= +k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 h1:Sztf7ESG9tAXRW/ACJZjrj5jhdOUqS2KFRQT+CTvu78= @@ -370,7 +370,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/e2e/infra/main.bicep b/hack/e2e/infra/main.bicep index a07e5984..ec44699d 100644 --- a/hack/e2e/infra/main.bicep +++ b/hack/e2e/infra/main.bicep @@ -236,6 +236,7 @@ output clusterFqdn string = aksCluster.properties.fqdn output msiVmName string = vmMsi.outputs.vmName output msiVmIp string = vmMsi.outputs.publicIpAddress +output msiVmPrivateIp string = vmMsi.outputs.privateIpAddress output msiVmPrincipalId string = vmMsi.outputs.principalId output tokenVmName string = vmToken.outputs.vmName diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index cc3d457b..d7ba0224 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -107,7 +107,7 @@ infra_deploy() { --query properties.outputs \ -o json) - local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_principal_id + local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_private_ip msi_vm_principal_id local token_vm_name token_vm_ip token_vm_private_ip offline_vm_name offline_vm_ip offline_vm_private_ip local kubeadm_vm_name kubeadm_vm_ip admin_username @@ -115,6 +115,7 @@ infra_deploy() { cluster_id=$(echo "${outputs}" | jq -r '.clusterId.value') msi_vm_name=$(echo "${outputs}" | jq -r '.msiVmName.value') msi_vm_ip=$(echo "${outputs}" | jq -r '.msiVmIp.value') + msi_vm_private_ip=$(echo "${outputs}" | jq -r '.msiVmPrivateIp.value // ""') msi_vm_principal_id=$(echo "${outputs}" | jq -r '.msiVmPrincipalId.value') token_vm_name=$(echo "${outputs}" | jq -r '.tokenVmName.value') token_vm_ip=$(echo "${outputs}" | jq -r '.tokenVmIp.value') @@ -126,6 +127,10 @@ infra_deploy() { kubeadm_vm_ip=$(echo "${outputs}" | jq -r '.kubeadmVmIp.value') admin_username=$(echo "${outputs}" | jq -r '.adminUsername.value') + if [[ -z "${msi_vm_private_ip}" ]] || ! is_valid_ipv4 "${msi_vm_private_ip}"; then + log_error "Missing or invalid MSI VM private IP from deployment outputs: '${msi_vm_private_ip}'" + return 1 + fi if [[ -z "${token_vm_private_ip}" ]] || ! is_valid_ipv4 "${token_vm_private_ip}"; then log_error "Missing or invalid token VM private IP from deployment outputs: '${token_vm_private_ip}'" return 1 @@ -140,6 +145,7 @@ infra_deploy() { state_set "cluster_id" "${cluster_id}" state_set "msi_vm_name" "${msi_vm_name}" state_set "msi_vm_ip" "${msi_vm_ip}" + state_set "msi_vm_private_ip" "${msi_vm_private_ip}" state_set "msi_vm_principal_id" "${msi_vm_principal_id}" state_set "token_vm_name" "${token_vm_name}" state_set "token_vm_ip" "${token_vm_ip}" diff --git a/hack/e2e/lib/node-join-msi.sh b/hack/e2e/lib/node-join-msi.sh index e9167574..6a94715e 100644 --- a/hack/e2e/lib/node-join-msi.sh +++ b/hack/e2e/lib/node-join-msi.sh @@ -14,6 +14,63 @@ readonly _E2E_NODE_JOIN_MSI_LOADED=1 # shellcheck disable=SC1091 source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +# --------------------------------------------------------------------------- +# prepare_localdns_host_resolver - Remove DHCP search/routing domains. +# +# Unbounded intentionally rejects systemd-resolved split-DNS layouts because +# flattening per-domain routing into one LocalDNS upstream list changes DNS +# semantics. Azure's Ubuntu image supplies a DHCP search domain by default, so +# put the disposable E2E host into the supported single-upstream layout before +# LocalDNS preflight runs. +# --------------------------------------------------------------------------- +prepare_localdns_host_resolver() { + local vm_ip="$1" + + log_info "Configuring the MSI host resolver for LocalDNS..." + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail + +interface="$(ip -4 route show default | awk 'NR == 1 { print $5 }')" +if [[ ! "${interface}" =~ ^[a-zA-Z0-9_.:-]+$ ]]; then + echo "could not determine a safe primary interface name: ${interface}" >&2 + exit 1 +fi + +cat > /etc/netplan/99-aks-flex-localdns.yaml <&2 + resolvectl domain >&2 + exit 1 +fi + +resolvectl domain +resolvectl dns "${interface}" +REMOTE +} + # --------------------------------------------------------------------------- # node_join_msi - Join the MSI VM # --------------------------------------------------------------------------- @@ -24,6 +81,8 @@ node_join_msi() { local vm_ip vm_ip="$(state_get msi_vm_ip)" + local vm_private_ip + vm_private_ip="$(state_get msi_vm_private_ip)" local cluster_id cluster_id="$(state_get cluster_id)" local subscription_id @@ -55,7 +114,8 @@ node_join_msi() { "node": { "kubelet": { "clusterFQDN": "${server_url}", - "caCertData": "${ca_cert_data}" + "caCertData": "${ca_cert_data}", + "nodeIP": "${vm_private_ip}" } }, "agent": { @@ -67,6 +127,35 @@ node_join_msi() { }, "requireMachineRegistration": true }, + "networking": { + "localDNS": { + "mode": "Required", + "vnetDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "PreferUDP", + "forwardDestination": "VnetDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + }, + "kubeDNSOverrides": { + ".": { + "queryLogging": "Error", + "protocol": "ForceTCP", + "forwardDestination": "ClusterCoreDNS", + "forwardPolicy": "Sequential", + "maxConcurrent": 1000, + "cacheDurationInSeconds": 3600, + "serveStaleDurationInSeconds": 3600, + "serveStale": "Immediate" + } + } + } + }, "components": { "kubernetes": "${E2E_KUBERNETES_VERSION}", "containerd": "${E2E_CONTAINERD_VERSION}", @@ -75,7 +164,10 @@ node_join_msi() { } EOF - # Step 2: Publish the AKS Machine goal and deploy the agent. + # Step 2: Put systemd-resolved into the layout supported by LocalDNS. + prepare_localdns_host_resolver "${vm_ip}" + + # Step 3: Publish the AKS Machine goal and deploy the agent. ensure_flex_controller machine_configmap_upsert "$(state_get msi_vm_name)" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}" _deploy_and_start_agent "${vm_ip}" "${config_file}" "aks-flex-node-msi" diff --git a/hack/e2e/lib/upgrade-drift.sh b/hack/e2e/lib/upgrade-drift.sh index f1be588f..bf6f6ffe 100644 --- a/hack/e2e/lib/upgrade-drift.sh +++ b/hack/e2e/lib/upgrade-drift.sh @@ -209,11 +209,61 @@ upgrade_drift_mode() { log_success "${mode} controller machine repave passed in $(timer_elapsed "${start}")s" } +localdns_disable_repave_msi() { + log_section "LocalDNS Disable Repave (MSI node)" + local desired_version settings_version vm_name vm_ip snapshot old_active_machine old_state old_version old_settings_version old_node_uid + desired_version="$(_cluster_current_kubernetes_version)" + settings_version="localdns-disabled-$(date +%s)" + vm_name="$(_mode_vm_name msi)" + vm_ip="$(_mode_vm_ip msi)" + + snapshot="$(_remote_active_machine_snapshot "${vm_ip}")" + IFS='|' read -r old_active_machine old_state old_version old_settings_version <<<"${snapshot}" + old_node_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" + + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail +config=/etc/aks-flex-node/config.json +tmp=$(mktemp /etc/aks-flex-node/config.json.XXXXXX) +jq '.networking.localDNS.mode = "Disabled"' "${config}" > "${tmp}" +chmod 0600 "${tmp}" +mv "${tmp}" "${config}" +systemctl restart aks-flex-node-agent.service +systemctl is-active --quiet aks-flex-node-agent.service +REMOTE + + _trigger_mode_repave msi "${desired_version}" "${settings_version}" + _wait_for_mode_repave msi "${desired_version}" "${settings_version}" "${old_active_machine}" "${old_node_uid}" + + local state_label cluster_dns + state_label="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/localdns-state}')" + cluster_dns="$(kubectl -n kube-system get service kube-dns -o jsonpath='{.spec.clusterIP}')" + if [[ "${state_label}" != "disabled" ]]; then + log_error "Node ${vm_name} localdns-state=${state_label}, expected disabled" + return 1 + fi + + remote_exec "${vm_ip}" "CLUSTER_DNS=${cluster_dns} sudo -E bash -s" <<'REMOTE' +set -euo pipefail +machine=$(machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}') +test -n "${machine}" +! systemd-run --quiet --pipe --wait --machine="${machine}" systemctl cat localdns.service >/dev/null 2>&1 +! ip link show localdns >/dev/null 2>&1 +! iptables -w -t raw -S | grep -q 'unbounded-localdns: skip conntrack' +systemd-run --quiet --pipe --wait --machine="${machine}" \ + grep -A1 -F 'clusterDNS:' /var/lib/kubelet/config.yaml \ + | grep -q -F -- "- ${CLUSTER_DNS}" +REMOTE + + log_success "MSI LocalDNS disable-through-repave validation passed" +} + upgrade_drift_all() { log_section "Controller Machine Repave (all modes)" upgrade_drift_mode msi upgrade_drift_mode token upgrade_drift_mode kubeadm + localdns_disable_repave_msi } upgrade_drift_msi() { upgrade_drift_mode msi; } diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 02ab9f6c..ac299d65 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -168,6 +168,80 @@ REMOTE } # --------------------------------------------------------------------------- +# validate_localdns_status - Verify nspawn LocalDNS on the selected VM. +validate_localdns_status() { + local vm_name="$1" + local vm_ip="$2" + log_info "Validating LocalDNS on ${vm_ip}..." + remote_exec "${vm_ip}" "sudo bash -s" <<'REMOTE' +set -euo pipefail +machine=$(sudo machinectl list --no-legend | awk '$1 ~ /^kube[12]$/ {print $1; exit}') +test -n "${machine}" +sudo systemd-run --quiet --pipe --wait --machine="${machine}" systemctl is-active --quiet localdns.service +sudo systemd-run --quiet --pipe --wait --machine="${machine}" \ + grep -qx 'nameserver 169.254.10.10' /etc/unbounded/localdns/resolv.conf +sudo systemd-run --quiet --pipe --wait --machine="${machine}" \ + systemctl cat kubelet.service | grep -q -- '--resolv-conf=/etc/unbounded/localdns/resolv.conf' +sudo ip address show dev localdns | grep -q '169.254.10.10/32' +sudo ip address show dev localdns | grep -q '169.254.10.11/32' +for chain in output prerouting; do + rules="$(sudo nft list chain ip unbounded_localdns "${chain}")" + for address in 169.254.10.10 169.254.10.11; do + for protocol in tcp udp; do + grep -Fq \ + "ip daddr ${address} ${protocol} dport 53 notrack comment \"unbounded-localdns: skip conntrack\"" \ + <<<"${rules}" + done + done +done +REMOTE + + local state_label + state_label="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/localdns-state}')" + if [[ "${state_label}" != "enabled" ]]; then + log_error "Node ${vm_name} localdns-state=${state_label}, expected enabled" + return 1 + fi + + local cluster_pod="localdns-clusterfirst-${vm_name}" + local default_pod="localdns-default-${vm_name}" + kubectl delete pod "${cluster_pod}" "${default_pod}" --ignore-not-found --wait=false >/dev/null + kubectl run "${cluster_pod}" --image=busybox:1.36 --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"${vm_name}\",\"dnsPolicy\":\"ClusterFirst\"}}" \ + --command -- nslookup kubernetes.default.svc.cluster.local >/dev/null + kubectl run "${default_pod}" --image=busybox:1.36 --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"${vm_name}\",\"dnsPolicy\":\"Default\"}}" \ + --command -- nslookup mcr.microsoft.com >/dev/null + + local pod + for pod in "${cluster_pod}" "${default_pod}"; do + local phase="" + for _ in $(seq 1 60); do + phase="$(kubectl get pod "${pod}" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + [[ "${phase}" == "Succeeded" || "${phase}" == "Failed" ]] && break + sleep 2 + done + if [[ "${phase}" != "Succeeded" ]]; then + kubectl describe pod "${pod}" >&2 || true + kubectl logs "${pod}" >&2 || true + log_error "LocalDNS query pod ${pod} ended in phase ${phase}" + return 1 + fi + done + + if ! kubectl logs "${cluster_pod}" | grep -Eq 'Server:[[:space:]]*169\.254\.10\.11'; then + log_error "ClusterFirst pod did not query LocalDNS cluster listener" + return 1 + fi + if ! kubectl logs "${default_pod}" | grep -Eq 'Server:[[:space:]]*169\.254\.10\.10'; then + log_error "Default-policy pod did not query LocalDNS node listener" + return 1 + fi + kubectl delete pod "${cluster_pod}" "${default_pod}" --ignore-not-found --wait=false >/dev/null + + log_success "LocalDNS validation passed on ${vm_ip}" +} + # validate_all_nodes - Check all MSI, token, offline, and kubeadm VMs joined # --------------------------------------------------------------------------- validate_all_nodes() { @@ -186,12 +260,13 @@ validate_all_nodes() { local msi_vm_name token_vm_name offline_vm_name kubeadm_vm_name local msi_vm_ip token_vm_ip offline_vm_ip kubeadm_vm_ip - local token_vm_private_ip offline_vm_private_ip + local msi_vm_private_ip token_vm_private_ip offline_vm_private_ip msi_vm_name="$(state_get msi_vm_name)" token_vm_name="$(state_get token_vm_name)" offline_vm_name="$(state_get offline_vm_name)" kubeadm_vm_name="$(state_get kubeadm_vm_name)" msi_vm_ip="$(state_get msi_vm_ip)" + msi_vm_private_ip="$(state_get msi_vm_private_ip)" token_vm_ip="$(state_get token_vm_ip)" offline_vm_ip="$(state_get offline_vm_ip)" kubeadm_vm_ip="$(state_get kubeadm_vm_ip)" @@ -203,9 +278,11 @@ validate_all_nodes() { validate_node_joined "${token_vm_name}" || failed=1 validate_node_joined "${offline_vm_name}" || failed=1 validate_node_joined "${kubeadm_vm_name}" || failed=1 + validate_node_ip "${msi_vm_name}" "${msi_vm_private_ip}" || failed=1 validate_node_ip "${token_vm_name}" "${token_vm_private_ip}" || failed=1 validate_node_ip "${offline_vm_name}" "${offline_vm_private_ip}" || failed=1 validate_npd_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 + validate_localdns_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 validate_npd_status "${token_vm_name}" "${token_vm_ip}" || failed=1 # TODO: re-enable once NPD is included in the upstream Unbounded bootstrap # artifact bundle and resolver used by offline artifact mode. diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index dfd302b3..d7d2c890 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "maps" agentconfig "github.com/Azure/unbounded/pkg/agent/config" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -26,6 +27,11 @@ const ( // // cfg.Node.Kubelet.ClusterFQDN and cfg.Node.Kubelet.CACertData must be populated. func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { + labels := maps.Clone(cfg.Node.Labels) + if labels == nil { + labels = map[string]string{} + } + ac := &agentconfig.AgentConfig{ MachineName: machineName, NodeName: cfg.Agent.NodeName, @@ -40,8 +46,9 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { Kubelet: agentconfig.AgentKubeletConfig{ ApiServer: cfg.APIServerURL(), NodeIP: cfg.Node.Kubelet.NodeIP, - Labels: cfg.Node.Labels, + Labels: labels, RegisterWithTaints: cfg.Node.Taints, + Configuration: kubeletConfiguration(cfg), }, CRI: agentconfig.CRIConfig{ Containerd: agentconfig.ContainerdConfig{ @@ -57,6 +64,26 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { }, } + if profile := cfg.Networking.LocalDNS; profile != nil { + corefile, _ := profile.CorefileTemplate() // Config validation runs before adaptation. + ac.LocalDNS = &agentconfig.AgentLocalDNSConfig{ + Enabled: profile.Enabled(), + CorefileTemplate: corefile, + } + state := "disabled" + if profile.Enabled() { + state = "enabled" + } + labels["kubernetes.azure.com/localdns-state"] = state + } + + if provider := cfg.Node.Kubelet.ImageCredentialProvider; provider != nil { + ac.Kubelet.ImageCredentialProvider = &agentconfig.ImageCredentialProvider{ + ConfigPath: provider.ConfigPath, + BinDir: provider.BinDir, + } + } + if cfg.Bootstrap.OfflineArtifacts.Source != "" { ac.OfflineArtifacts = &agentconfig.AgentOfflineArtifacts{ Source: cfg.Bootstrap.OfflineArtifacts.Source, @@ -103,6 +130,17 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { return ac } +func kubeletConfiguration(cfg *Config) map[string]any { + return map[string]any{ + "maxPods": cfg.Node.MaxPods, + "imageGCHighThresholdPercent": cfg.Node.Kubelet.ImageGCHighThreshold, + "imageGCLowThresholdPercent": cfg.Node.Kubelet.ImageGCLowThreshold, + "logging": map[string]any{ + "verbosity": cfg.Node.Kubelet.Verbosity, + }, + } +} + // ResolveMachineGoalState converts FlexNode config to the shared agent config // and resolves the nspawn machine goal state. Bootstrap and preflight both use // this helper so preflight validates the same sources that bootstrap consumes. diff --git a/pkg/config/adapter_test.go b/pkg/config/adapter_test.go index 510af5e4..82bdf6e5 100644 --- a/pkg/config/adapter_test.go +++ b/pkg/config/adapter_test.go @@ -19,12 +19,20 @@ func TestToAgentConfig_BootstrapToken(t *testing.T) { Components: ComponentsConfig{Kubernetes: "1.30.0"}, Networking: NetworkingConfig{DNSServiceIP: "10.0.0.10"}, Node: NodeConfig{ - Labels: map[string]string{"env": "test"}, - Taints: []string{"dedicated=infra:NoSchedule"}, + MaxPods: 42, + Labels: map[string]string{"env": "test"}, + Taints: []string{"dedicated=infra:NoSchedule"}, Kubelet: KubeletConfig{ - ClusterFQDN: "api.example.com:6443", - CACertData: "dGVzdC1jYS1kYXRh", - NodeIP: "10.225.0.4", + Verbosity: 4, + ImageGCHighThreshold: 90, + ImageGCLowThreshold: 75, + ClusterFQDN: "api.example.com:6443", + CACertData: "dGVzdC1jYS1kYXRh", + NodeIP: "10.225.0.4", + ImageCredentialProvider: &ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, }, }, } @@ -65,6 +73,31 @@ func TestToAgentConfig_BootstrapToken(t *testing.T) { if len(ac.Kubelet.RegisterWithTaints) != 1 || ac.Kubelet.RegisterWithTaints[0] != "dedicated=infra:NoSchedule" { t.Fatalf("Kubelet.RegisterWithTaints=%v, want [dedicated=infra:NoSchedule]", ac.Kubelet.RegisterWithTaints) } + if got := ac.Kubelet.Configuration["maxPods"]; got != 42 { + t.Fatalf("Kubelet.Configuration.maxPods=%v, want 42", got) + } + if got := ac.Kubelet.Configuration["imageGCHighThresholdPercent"]; got != 90 { + t.Fatalf("Kubelet.Configuration.imageGCHighThresholdPercent=%v, want 90", got) + } + if got := ac.Kubelet.Configuration["imageGCLowThresholdPercent"]; got != 75 { + t.Fatalf("Kubelet.Configuration.imageGCLowThresholdPercent=%v, want 75", got) + } + logging, ok := ac.Kubelet.Configuration["logging"].(map[string]any) + if !ok { + t.Fatalf("Kubelet.Configuration.logging=%T, want map[string]any", ac.Kubelet.Configuration["logging"]) + } + if logging["verbosity"] != 4 { + t.Fatalf("Kubelet.Configuration.logging=%v, want verbosity=4", logging) + } + if ac.Kubelet.ImageCredentialProvider == nil { + t.Fatal("Kubelet.ImageCredentialProvider=nil, want provider") + } + if ac.Kubelet.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" { + t.Fatalf("Kubelet.ImageCredentialProvider.ConfigPath=%q", ac.Kubelet.ImageCredentialProvider.ConfigPath) + } + if ac.Kubelet.ImageCredentialProvider.BinDir != "/usr/local/lib/kubelet-credential-providers" { + t.Fatalf("Kubelet.ImageCredentialProvider.BinDir=%q", ac.Kubelet.ImageCredentialProvider.BinDir) + } } func TestToAgentConfig_NodeName(t *testing.T) { diff --git a/pkg/config/config.go b/pkg/config/config.go index ecc670c5..9cbb376f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -251,12 +251,24 @@ type KubeletConfig struct { ClusterFQDN string `json:"clusterFQDN,omitempty"` // Kubernetes API server FQDN from AKS RP bootstrap data CACertData string `json:"caCertData"` // Base64-encoded CA certificate data NodeIP string `json:"nodeIP"` // IP address to advertise as the node's primary IP (--node-ip kubelet flag) + + // ImageCredentialProvider configures kubelet's exec image credential + // provider. The referenced paths are inside the nspawn machine. + ImageCredentialProvider *ImageCredentialProviderConfig `json:"imageCredentialProvider,omitempty"` +} + +// ImageCredentialProviderConfig identifies the exec image credential provider +// configuration and binary directory inside the nspawn machine. +type ImageCredentialProviderConfig struct { + ConfigPath string `json:"configPath"` + BinDir string `json:"binDir"` } // NetworkingConfig is the AKS RP networking contract used by the agent at runtime. type NetworkingConfig struct { - DNSServiceIP string `json:"dnsServiceIP,omitempty"` // Cluster DNS service IP (default: 10.0.0.10 for AKS) - CNIVersion string `json:"cniVersion,omitempty"` + DNSServiceIP string `json:"dnsServiceIP,omitempty"` // Cluster DNS service IP (default: 10.0.0.10 for AKS) + CNIVersion string `json:"cniVersion,omitempty"` + LocalDNS *LocalDNSProfile `json:"localDNS,omitempty"` } // NPDConfig holds configuration settings for the Node Problem Detector (NPD). @@ -882,6 +894,12 @@ func (c *Config) validate() error { if err := c.Bootstrap.validate(); err != nil { return err } + if err := c.Networking.LocalDNS.Validate(); err != nil { + return fmt.Errorf("invalid networking.localDNS: %w", err) + } + if err := c.Node.Kubelet.validate(); err != nil { + return err + } if err := c.validateAuthSettings(); err != nil { return err @@ -893,6 +911,20 @@ func (c *Config) validate() error { return nil } +func (c *KubeletConfig) validate() error { + kubelet := agentconfig.AgentKubeletConfig{} + if c.ImageCredentialProvider != nil { + kubelet.ImageCredentialProvider = &agentconfig.ImageCredentialProvider{ + ConfigPath: c.ImageCredentialProvider.ConfigPath, + BinDir: c.ImageCredentialProvider.BinDir, + } + } + if err := kubelet.Validate(); err != nil { + return fmt.Errorf("invalid node.kubelet configuration: %w", err) + } + return nil +} + func (c *Config) validateAuthSettings() error { armAuthMethodCount := 0 for _, m := range []bool{c.IsARCEnabled(), c.IsSPConfigured(), c.IsMIConfigured()} { diff --git a/pkg/config/copy_test.go b/pkg/config/copy_test.go index cae05009..114b0a98 100644 --- a/pkg/config/copy_test.go +++ b/pkg/config/copy_test.go @@ -17,6 +17,12 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) { Node: NodeConfig{ Labels: map[string]string{"l": "1"}, Taints: []string{"dedicated=infra:NoSchedule"}, + Kubelet: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, + }, }, } @@ -47,6 +53,9 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) { if cfg.Components.Gantry == nil || copy.Components.Gantry == nil || cfg.Components.Gantry == copy.Components.Gantry { t.Fatalf("Gantry pointer shared or nil") } + if cfg.Node.Kubelet.ImageCredentialProvider == nil || copy.Node.Kubelet.ImageCredentialProvider == nil || cfg.Node.Kubelet.ImageCredentialProvider == copy.Node.Kubelet.ImageCredentialProvider { + t.Fatalf("ImageCredentialProvider pointer shared or nil") + } // Maps should not be shared (validate via independent mutation behavior). cfg.Azure.Arc.Tags["k"] = "orig" @@ -67,6 +76,11 @@ func TestConfigDeepCopy_DoesNotSharePointersOrMaps(t *testing.T) { t.Fatalf("Node.Labels shared; orig=%q, want %q", cfg.Node.Labels["l"], "orig") } + copy.Node.Kubelet.ImageCredentialProvider.ConfigPath = "/etc/kubernetes/other.yaml" + if cfg.Node.Kubelet.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" { + t.Fatal("ImageCredentialProvider shared between config copies") + } + // Taints slice should not be shared. if len(copy.Node.Taints) != 1 || copy.Node.Taints[0] != "dedicated=infra:NoSchedule" { t.Fatalf("Node.Taints copy=%v, want [dedicated=infra:NoSchedule]", copy.Node.Taints) diff --git a/pkg/config/internal/localdns/assets/localdns.corefile.tmpl b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl new file mode 100644 index 00000000..cbb86f94 --- /dev/null +++ b/pkg/config/internal/localdns/assets/localdns.corefile.tmpl @@ -0,0 +1,25 @@ +health-check.localdns.local:53 { + bind {{.NodeListener}} {{.ClusterListener}} + whoami +} + +{{range .Blocks}}{{.Zone}}:53 { + {{if .LogQueries}}log{{else}}errors{{end}} + bind {{.Listener}} + forward . {{.Upstream}} { + {{if .ForceTCP}}force_tcp + {{end}}policy {{.ForwardPolicy}} + max_concurrent {{.MaxConcurrent}} + } + ready {{.Listener}}:8181 + cache {{.CacheDuration}} { + success 9984 + denial 9984 + {{if .ServeStale}}serve_stale {{.ServeStaleDuration}}s {{.ServeStalePolicy}} + {{end}}servfail 0 + } + loop + prometheus {{$.MetricsAddress}} +} + +{{end}} \ No newline at end of file diff --git a/pkg/config/internal/localdns/localdns.go b/pkg/config/internal/localdns/localdns.go new file mode 100644 index 00000000..85dd1ce1 --- /dev/null +++ b/pkg/config/internal/localdns/localdns.go @@ -0,0 +1,229 @@ +package localdns + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "sort" + "strings" + "text/template" +) + +const ( + LocalDNSModeRequired = "Required" + LocalDNSModePreferred = "Preferred" + LocalDNSModeDisabled = "Disabled" +) + +// LocalDNSProfile mirrors the AKS node-pool LocalDNS configuration contract. +// Keep rendering behavior aligned with AgentBaker: +// https://github.com/Azure/AgentBaker/blob/62a783b7a967352bb4726e636d22930c9973f4f7/aks-node-controller/parser/templates/localdns.toml.gtpl +type LocalDNSProfile struct { + Mode string `json:"mode"` + VnetDNSOverrides map[string]LocalDNSOverride `json:"vnetDNSOverrides,omitempty"` + KubeDNSOverrides map[string]LocalDNSOverride `json:"kubeDNSOverrides,omitempty"` +} + +// LocalDNSOverride configures one CoreDNS server block. +type LocalDNSOverride struct { + QueryLogging string `json:"queryLogging,omitempty"` + Protocol string `json:"protocol,omitempty"` + ForwardDestination string `json:"forwardDestination,omitempty"` + ForwardPolicy string `json:"forwardPolicy,omitempty"` + MaxConcurrent int `json:"maxConcurrent,omitempty"` + CacheDurationInSeconds int `json:"cacheDurationInSeconds,omitempty"` + ServeStaleDurationInSeconds int `json:"serveStaleDurationInSeconds,omitempty"` + ServeStale string `json:"serveStale,omitempty"` +} + +// Validate verifies the AKS LocalDNS profile values accepted by the renderer. +func (p *LocalDNSProfile) Validate() error { + if p == nil { + return nil + } + if p.Mode != LocalDNSModeRequired && p.Mode != LocalDNSModePreferred && p.Mode != LocalDNSModeDisabled { + return fmt.Errorf("mode must be Required, Preferred, or Disabled") + } + var errs []error + for class, overrides := range map[string]map[string]LocalDNSOverride{ + "vnetDNSOverrides": p.VnetDNSOverrides, + "kubeDNSOverrides": p.KubeDNSOverrides, + } { + for zone, override := range overrides { + if strings.TrimSpace(zone) == "" || strings.ContainsAny(zone, "{} \t\r\n") { + errs = append(errs, fmt.Errorf("%s zone %q is invalid", class, zone)) + } + if err := override.validate(); err != nil { + errs = append(errs, fmt.Errorf("%s[%q]: %w", class, zone, err)) + } + } + } + return errors.Join(errs...) +} + +func (o LocalDNSOverride) validate() error { + var errs []error + if o.QueryLogging != "" && o.QueryLogging != "Error" && o.QueryLogging != "Log" { + errs = append(errs, fmt.Errorf("queryLogging must be Error or Log")) + } + if o.Protocol != "" && o.Protocol != "PreferUDP" && o.Protocol != "ForceTCP" { + errs = append(errs, fmt.Errorf("protocol must be PreferUDP or ForceTCP")) + } + if o.ForwardDestination != "" && o.ForwardDestination != "VnetDNS" && o.ForwardDestination != "ClusterCoreDNS" { + errs = append(errs, fmt.Errorf("forwardDestination must be VnetDNS or ClusterCoreDNS")) + } + if o.ForwardPolicy != "" && o.ForwardPolicy != "Sequential" && o.ForwardPolicy != "RoundRobin" && o.ForwardPolicy != "Random" { + errs = append(errs, fmt.Errorf("forwardPolicy must be Sequential, RoundRobin, or Random")) + } + if o.ServeStale != "" && o.ServeStale != "Disable" && o.ServeStale != "Verify" && o.ServeStale != "Immediate" { + errs = append(errs, fmt.Errorf("serveStale must be Disable, Verify, or Immediate")) + } + for name, value := range map[string]int{ + "maxConcurrent": o.MaxConcurrent, + "cacheDurationInSeconds": o.CacheDurationInSeconds, + "serveStaleDurationInSeconds": o.ServeStaleDurationInSeconds, + } { + if value < 0 { + errs = append(errs, fmt.Errorf("%s must not be negative", name)) + } + } + return errors.Join(errs...) +} + +// Enabled reports whether the profile requires LocalDNS installation. +func (p *LocalDNSProfile) Enabled() bool { + return p != nil && p.Mode == LocalDNSModeRequired +} + +//go:embed assets/localdns.corefile.tmpl +var aksLocalDNSCorefileTemplate string + +type localDNSCorefileData struct { + NodeListener string + ClusterListener string + MetricsAddress string + Blocks []localDNSCorefileBlock +} + +type localDNSCorefileBlock struct { + Zone string + Listener string + Upstream string + LogQueries bool + ForceTCP bool + ForwardPolicy string + MaxConcurrent int + CacheDuration int + ServeStale bool + ServeStaleDuration int + ServeStalePolicy string +} + +// CorefileTemplate renders AKS LocalDNS policy into an Unbounded Corefile template. +func (p *LocalDNSProfile) CorefileTemplate() (string, error) { + if p == nil { + return "", nil + } + if err := p.Validate(); err != nil { + return "", err + } + if !p.Enabled() { + return "", nil + } + + vnet := p.VnetDNSOverrides + if len(vnet) == 0 { + vnet = map[string]LocalDNSOverride{".": {ForwardDestination: "VnetDNS"}} + } + kube := p.KubeDNSOverrides + if len(kube) == 0 { + kube = map[string]LocalDNSOverride{".": {ForwardDestination: "ClusterCoreDNS"}} + } + + data := localDNSCorefileData{ + NodeListener: "{{ .NodeListenerIP }}", + ClusterListener: "{{ .ClusterListenerIP }}", + MetricsAddress: "{{ .MetricsAddress }}", + Blocks: append(localDNSCorefileBlocks(vnet, "{{ .NodeListenerIP }}", "VnetDNS"), localDNSCorefileBlocks(kube, "{{ .ClusterListenerIP }}", "ClusterCoreDNS")...), + } + tmpl, err := template.New("aks-localdns-corefile").Parse(aksLocalDNSCorefileTemplate) + if err != nil { + return "", fmt.Errorf("parse AKS LocalDNS Corefile template: %w", err) + } + var out bytes.Buffer + if err := tmpl.Execute(&out, data); err != nil { + return "", fmt.Errorf("render AKS LocalDNS Corefile template: %w", err) + } + return out.String(), nil +} + +func localDNSCorefileBlocks(overrides map[string]LocalDNSOverride, listener, defaultDestination string) []localDNSCorefileBlock { + zones := make([]string, 0, len(overrides)) + for zone := range overrides { + zones = append(zones, zone) + } + sort.Strings(zones) + + blocks := make([]localDNSCorefileBlock, 0, len(zones)) + for _, zone := range zones { + o := withLocalDNSDefaults(overrides[zone], defaultDestination) + upstream := "{{ .NodeUpstreamIPsJoined }}" + if o.ForwardDestination == "ClusterCoreDNS" { + upstream = "{{ .ClusterDNSServiceIP }}" + } + blocks = append(blocks, localDNSCorefileBlock{ + Zone: zone, + Listener: listener, + Upstream: upstream, + LogQueries: o.QueryLogging == "Log", + ForceTCP: o.Protocol == "ForceTCP", + ForwardPolicy: localDNSForwardPolicy(o.ForwardPolicy), + MaxConcurrent: o.MaxConcurrent, + CacheDuration: o.CacheDurationInSeconds, + ServeStale: o.ServeStale != "Disable", + ServeStaleDuration: o.ServeStaleDurationInSeconds, + ServeStalePolicy: strings.ToLower(o.ServeStale), + }) + } + return blocks +} + +func withLocalDNSDefaults(o LocalDNSOverride, destination string) LocalDNSOverride { + if o.QueryLogging == "" { + o.QueryLogging = "Error" + } + if o.Protocol == "" { + o.Protocol = "ForceTCP" + } + if o.ForwardDestination == "" { + o.ForwardDestination = destination + } + if o.ForwardPolicy == "" { + o.ForwardPolicy = "Sequential" + } + if o.MaxConcurrent == 0 { + o.MaxConcurrent = 1000 + } + if o.CacheDurationInSeconds == 0 { + o.CacheDurationInSeconds = 3600 + } + if o.ServeStaleDurationInSeconds == 0 { + o.ServeStaleDurationInSeconds = 3600 + } + if o.ServeStale == "" { + o.ServeStale = "Immediate" + } + return o +} + +func localDNSForwardPolicy(policy string) string { + switch policy { + case "RoundRobin": + return "round_robin" + case "Random": + return "random" + default: + return "sequential" + } +} diff --git a/pkg/config/internal/localdns/localdns_test.go b/pkg/config/internal/localdns/localdns_test.go new file mode 100644 index 00000000..536f1e85 --- /dev/null +++ b/pkg/config/internal/localdns/localdns_test.go @@ -0,0 +1,52 @@ +package localdns + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCorefileTemplateGolden(t *testing.T) { + t.Parallel() + + profile := &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + VnetDNSOverrides: map[string]LocalDNSOverride{ + ".": { + QueryLogging: "Log", Protocol: "PreferUDP", ForwardDestination: "VnetDNS", + ForwardPolicy: "Sequential", MaxConcurrent: 1000, CacheDurationInSeconds: 30, + ServeStaleDurationInSeconds: 60, ServeStale: "Immediate", + }, + "cluster.local": { + QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", + ForwardPolicy: "RoundRobin", MaxConcurrent: 2000, CacheDurationInSeconds: 45, + ServeStale: "Disable", + }, + }, + KubeDNSOverrides: map[string]LocalDNSOverride{ + ".": { + QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", + ForwardPolicy: "Random", MaxConcurrent: 1500, CacheDurationInSeconds: 90, + ServeStaleDurationInSeconds: 120, ServeStale: "Verify", + }, + }, + } + + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatalf("CorefileTemplate() error = %v", err) + } + goldenPath := filepath.Join("testdata", "localdns.corefile.golden") + if os.Getenv("UPDATE_GOLDEN") == "1" { + if err := os.WriteFile(goldenPath, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(goldenPath) + if err != nil { + t.Fatal(err) + } + if got != string(want) { + t.Fatalf("Corefile mismatch (-want +got):\n--- want ---\n%s\n--- got ---\n%s", want, got) + } +} diff --git a/pkg/config/internal/localdns/testdata/localdns.corefile.golden b/pkg/config/internal/localdns/testdata/localdns.corefile.golden new file mode 100644 index 00000000..c1dbcd8d --- /dev/null +++ b/pkg/config/internal/localdns/testdata/localdns.corefile.golden @@ -0,0 +1,60 @@ +health-check.localdns.local:53 { + bind {{ .NodeListenerIP }} {{ .ClusterListenerIP }} + whoami +} + +.:53 { + log + bind {{ .NodeListenerIP }} + forward . {{ .NodeUpstreamIPsJoined }} { + policy sequential + max_concurrent 1000 + } + ready {{ .NodeListenerIP }}:8181 + cache 30 { + success 9984 + denial 9984 + serve_stale 60s immediate + servfail 0 + } + loop + prometheus {{ .MetricsAddress }} +} + +cluster.local:53 { + errors + bind {{ .NodeListenerIP }} + forward . {{ .ClusterDNSServiceIP }} { + force_tcp + policy round_robin + max_concurrent 2000 + } + ready {{ .NodeListenerIP }}:8181 + cache 45 { + success 9984 + denial 9984 + servfail 0 + } + loop + prometheus {{ .MetricsAddress }} +} + +.:53 { + errors + bind {{ .ClusterListenerIP }} + forward . {{ .ClusterDNSServiceIP }} { + force_tcp + policy random + max_concurrent 1500 + } + ready {{ .ClusterListenerIP }}:8181 + cache 90 { + success 9984 + denial 9984 + serve_stale 120s verify + servfail 0 + } + loop + prometheus {{ .MetricsAddress }} +} + diff --git a/pkg/config/kubelet_config_test.go b/pkg/config/kubelet_config_test.go new file mode 100644 index 00000000..c2692118 --- /dev/null +++ b/pkg/config/kubelet_config_test.go @@ -0,0 +1,82 @@ +package config + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestKubeletConfigJSON(t *testing.T) { + t.Parallel() + + data := []byte(`{ + "imageCredentialProvider": { + "configPath": "/etc/kubernetes/credential-provider.yaml", + "binDir": "/usr/local/lib/kubelet-credential-providers" + } + }`) + var config KubeletConfig + if err := json.Unmarshal(data, &config); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if config.ImageCredentialProvider == nil { + t.Fatal("ImageCredentialProvider=nil, want provider") + } + if config.ImageCredentialProvider.ConfigPath != "/etc/kubernetes/credential-provider.yaml" || config.ImageCredentialProvider.BinDir != "/usr/local/lib/kubelet-credential-providers" { + t.Fatalf("ImageCredentialProvider=%#v", config.ImageCredentialProvider) + } +} + +func TestKubeletConfigValidate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + config KubeletConfig + wantErr string + }{ + {name: "empty"}, + { + name: "image credential provider", + config: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "/usr/local/lib/kubelet-credential-providers", + }, + }, + }, + { + name: "missing provider config path", + config: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{BinDir: "/usr/bin"}, + }, + wantErr: "ConfigPath", + }, + { + name: "relative provider binary directory", + config: KubeletConfig{ + ImageCredentialProvider: &ImageCredentialProviderConfig{ + ConfigPath: "/etc/kubernetes/credential-provider.yaml", + BinDir: "bin", + }, + }, + wantErr: "absolute path", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := tt.config.validate() + if tt.wantErr == "" { + if err != nil { + t.Fatalf("validate: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("validate error=%v, want substring %q", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/config/localdns.go b/pkg/config/localdns.go new file mode 100644 index 00000000..0422fbfe --- /dev/null +++ b/pkg/config/localdns.go @@ -0,0 +1,15 @@ +package config + +import "github.com/Azure/AKSFlexNode/pkg/config/internal/localdns" + +const ( + LocalDNSModeRequired = localdns.LocalDNSModeRequired + LocalDNSModePreferred = localdns.LocalDNSModePreferred + LocalDNSModeDisabled = localdns.LocalDNSModeDisabled +) + +// LocalDNSProfile is the AKS node-pool LocalDNS configuration contract. +type LocalDNSProfile = localdns.LocalDNSProfile + +// LocalDNSOverride configures one AKS LocalDNS server block. +type LocalDNSOverride = localdns.LocalDNSOverride diff --git a/pkg/config/localdns_adapter_test.go b/pkg/config/localdns_adapter_test.go new file mode 100644 index 00000000..20fbdb68 --- /dev/null +++ b/pkg/config/localdns_adapter_test.go @@ -0,0 +1,53 @@ +package config + +import ( + "strings" + "testing" +) + +func TestToAgentConfigLocalDNS(t *testing.T) { + t.Parallel() + + cfg := &Config{ + Networking: NetworkingConfig{ + DNSServiceIP: "10.0.0.10", + LocalDNS: &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + KubeDNSOverrides: map[string]LocalDNSOverride{ + ".": {ForwardDestination: "ClusterCoreDNS"}, + }, + }, + }, + Node: NodeConfig{Labels: map[string]string{"example": "value"}}, + } + + got := ToAgentConfig(cfg, "kube1") + if got.LocalDNS == nil || !got.LocalDNS.Enabled { + t.Fatalf("LocalDNS = %#v, want enabled", got.LocalDNS) + } + if !strings.Contains(got.LocalDNS.CorefileTemplate, "{{ .ClusterDNSServiceIP }}") { + t.Fatalf("CorefileTemplate missing cluster DNS placeholder:\n%s", got.LocalDNS.CorefileTemplate) + } + if got.Kubelet.Labels["kubernetes.azure.com/localdns-state"] != "enabled" { + t.Fatalf("LocalDNS node label missing: %#v", got.Kubelet.Labels) + } + if got.Cluster.ClusterDNS != "10.0.0.10" { + t.Fatalf("ClusterDNS = %q, want original service IP", got.Cluster.ClusterDNS) + } +} + +func TestToAgentConfigDisabledLocalDNS(t *testing.T) { + t.Parallel() + + cfg := &Config{Networking: NetworkingConfig{ + DNSServiceIP: "10.0.0.10", + LocalDNS: &LocalDNSProfile{Mode: LocalDNSModeDisabled}, + }} + got := ToAgentConfig(cfg, "kube1") + if got.LocalDNS == nil || got.LocalDNS.Enabled { + t.Fatalf("LocalDNS = %#v, want explicitly disabled", got.LocalDNS) + } + if got.Kubelet.Labels["kubernetes.azure.com/localdns-state"] != "disabled" { + t.Fatalf("LocalDNS disabled node label missing: %#v", got.Kubelet.Labels) + } +} diff --git a/pkg/config/localdns_test.go b/pkg/config/localdns_test.go new file mode 100644 index 00000000..81524ed1 --- /dev/null +++ b/pkg/config/localdns_test.go @@ -0,0 +1,143 @@ +package config + +import ( + "strings" + "testing" +) + +func TestLocalDNSProfileCorefileTemplate(t *testing.T) { + t.Parallel() + + profile := &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + VnetDNSOverrides: map[string]LocalDNSOverride{ + ".": {ForwardDestination: "VnetDNS", Protocol: "PreferUDP"}, + }, + KubeDNSOverrides: map[string]LocalDNSOverride{ + ".": {ForwardDestination: "ClusterCoreDNS", Protocol: "ForceTCP", ForwardPolicy: "RoundRobin"}, + }, + } + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatalf("CorefileTemplate() error = %v", err) + } + for _, want := range []string{ + "bind {{ .NodeListenerIP }}", + "forward . {{ .NodeUpstreamIPsJoined }}", + "bind {{ .ClusterListenerIP }}", + "forward . {{ .ClusterDNSServiceIP }}", + "force_tcp", + "policy round_robin", + } { + if !strings.Contains(got, want) { + t.Fatalf("CorefileTemplate() missing %q:\n%s", want, got) + } + } +} + +func TestLocalDNSProfileCorefileOptions(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + override LocalDNSOverride + want []string + wantAbsent []string + }{ + { + name: "VnetDNS PreferUDP sequential immediate", + override: LocalDNSOverride{QueryLogging: "Log", Protocol: "PreferUDP", ForwardDestination: "VnetDNS", ForwardPolicy: "Sequential", MaxConcurrent: 1000, CacheDurationInSeconds: 30, ServeStaleDurationInSeconds: 60, ServeStale: "Immediate"}, + want: []string{"log", "forward . {{ .NodeUpstreamIPsJoined }}", "policy sequential", "max_concurrent 1000", "cache 30", "serve_stale 60s immediate"}, + wantAbsent: []string{"force_tcp"}, + }, + { + name: "ClusterCoreDNS ForceTCP round robin verify", + override: LocalDNSOverride{QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", ForwardPolicy: "RoundRobin", ServeStale: "Verify"}, + want: []string{"errors", "forward . {{ .ClusterDNSServiceIP }}", "force_tcp", "policy round_robin", "serve_stale 3600s verify"}, + }, + { + name: "random without stale", + override: LocalDNSOverride{ForwardDestination: "ClusterCoreDNS", ForwardPolicy: "Random", ServeStale: "Disable"}, + want: []string{"policy random"}, + wantAbsent: []string{"serve_stale"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + profile := &LocalDNSProfile{ + Mode: LocalDNSModeRequired, + VnetDNSOverrides: map[string]LocalDNSOverride{".": test.override}, + KubeDNSOverrides: map[string]LocalDNSOverride{".": test.override}, + } + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatalf("CorefileTemplate() error = %v", err) + } + for _, want := range test.want { + if !strings.Contains(got, want) { + t.Errorf("CorefileTemplate() missing %q:\n%s", want, got) + } + } + for _, unwanted := range test.wantAbsent { + if strings.Contains(got, unwanted) { + t.Errorf("CorefileTemplate() unexpectedly contains %q:\n%s", unwanted, got) + } + } + }) + } +} + +func TestLocalDNSProfileCorefileZoneOrder(t *testing.T) { + t.Parallel() + + profile := &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{ + "z.example": {}, + ".": {}, + "a.example": {}, + }} + got, err := profile.CorefileTemplate() + if err != nil { + t.Fatal(err) + } + root := strings.Index(got, ".:53 {") + a := strings.Index(got, "a.example:53 {") + z := strings.Index(got, "z.example:53 {") + if root < 0 || a < root || z < a { + t.Fatalf("zones are not deterministic: root=%d a=%d z=%d\n%s", root, a, z, got) + } +} + +func TestLocalDNSProfileValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + profile *LocalDNSProfile + wantErr string + }{ + {name: "required", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired}}, + {name: "preferred", profile: &LocalDNSProfile{Mode: LocalDNSModePreferred}}, + {name: "disabled", profile: &LocalDNSProfile{Mode: LocalDNSModeDisabled}}, + {name: "invalid mode", profile: &LocalDNSProfile{Mode: "On"}, wantErr: "mode"}, + {name: "invalid protocol", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {Protocol: "TLS"}}}, wantErr: "protocol"}, + {name: "invalid logging", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {QueryLogging: "Debug"}}}, wantErr: "queryLogging"}, + {name: "invalid destination", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ForwardDestination: "External"}}}, wantErr: "forwardDestination"}, + {name: "invalid policy", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ForwardPolicy: "First"}}}, wantErr: "forwardPolicy"}, + {name: "invalid stale", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {ServeStale: "Always"}}}, wantErr: "serveStale"}, + {name: "negative cache", profile: &LocalDNSProfile{Mode: LocalDNSModeRequired, VnetDNSOverrides: map[string]LocalDNSOverride{".": {CacheDurationInSeconds: -1}}}, wantErr: "cacheDurationInSeconds"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + err := test.profile.Validate() + if test.wantErr == "" && err != nil { + t.Fatalf("validate() error = %v", err) + } + if test.wantErr != "" && (err == nil || !strings.Contains(err.Error(), test.wantErr)) { + t.Fatalf("validate() error = %v, want %q", err, test.wantErr) + } + }) + } +}