diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 1685b6262..56050e44d 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -1105,6 +1105,27 @@ func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing { } } +// dualStackAdditionalAddresses returns the IPv6 half of a dual-stack ingress +// listener, to pair with a primary 0.0.0.0 socket on the same port. Ipv4Compat +// stays false: clearing IPV6_V6ONLY would collide with that primary. +func dualStackAdditionalAddresses(port uint32) []*listenerv3.AdditionalAddress { + return []*listenerv3.AdditionalAddress{ + { + Address: &corev3.Address{ + Address: &corev3.Address_SocketAddress{ + SocketAddress: &corev3.SocketAddress{ + Address: "::", + Ipv4Compat: false, + PortSpecifier: &corev3.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + }, + } +} + func (x *XdsServer) buildListener() *listenerv3.Listener { hcm := x.buildHcm("ingress_http", true) @@ -1120,6 +1141,7 @@ func (x *XdsServer) buildListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.ingressPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ @@ -1179,6 +1201,7 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.httpsPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 8580685cb..e40ece3ad 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -158,6 +158,22 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { if sa.GetAddress() != "0.0.0.0" { t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) } + + addrs := l.GetAdditionalAddresses() + if len(addrs) == 0 { + t.Fatalf("Expected an additional address on %s, got none", IngressHTTPListener) + } + + asa := addrs[0].GetAddress().GetSocketAddress() + if asa.GetAddress() != "::" { + t.Errorf("Expected additional address '::', got %s", asa.GetAddress()) + } + if asa.GetIpv4Compat() { + t.Errorf("Expected additional address Ipv4Compat to be false") + } + if asa.GetPortValue() != 8081 { + t.Errorf("Expected additional port 8081, got %d", asa.GetPortValue()) + } } } @@ -196,6 +212,25 @@ func TestXdsServer_UpdateSnapshot_WithHttps(t *testing.T) { if sa.GetPortValue() != 8443 { t.Errorf("Expected port 8443, got %d", sa.GetPortValue()) } + if sa.GetAddress() != "0.0.0.0" { + t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) + } + + addrs := l.GetAdditionalAddresses() + if len(addrs) == 0 { + t.Fatalf("Expected an additional address on %s, got none", IngressHTTPSListener) + } + + asa := addrs[0].GetAddress().GetSocketAddress() + if asa.GetAddress() != "::" { + t.Errorf("Expected additional address '::', got %s", asa.GetAddress()) + } + if asa.GetIpv4Compat() { + t.Errorf("Expected additional address Ipv4Compat to be false") + } + if asa.GetPortValue() != 8443 { + t.Errorf("Expected additional port 8443, got %d", asa.GetPortValue()) + } // Verify the TLS config references the serving cert via SDS rather // than embedding it: inline filename DataSources are read only once diff --git a/hack/check-atenet-ip-families.sh b/hack/check-atenet-ip-families.sh new file mode 100755 index 000000000..569d6b680 --- /dev/null +++ b/hack/check-atenet-ip-families.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Preconditions: +# IP_FAMILY=ipv4|dual|ipv6 hack/create-kind-cluster.sh +# hack/install-ate-kind.sh --deploy-ate-system +# +# Checks that both atenet gateways bind and serve on every IP family the +# cluster has. The assertions are read off the cluster, so one run is valid on +# all three families -- worth doing on each, because they fail differently: on +# dual-stack a v4-wildcard bind is a silent data-path gap (the Service has an +# IPv6 ClusterIP nothing listens on), on IPv6-only it is a crashloop, because +# the kubelet probes the pod on its only address. +set -o errexit -o nounset -o pipefail +ROOT="$(git rev-parse --show-toplevel)"; cd "${ROOT}" + +CTX="${KUBECTL_CONTEXT:-kind-kind}" +K="kubectl --context ${CTX}" +NS="${NS:-ate-system}" +PROBE_IMAGE="${PROBE_IMAGE:-busybox:1.36}" +PROBE_POD="atenet-ipfamily-probe" + +fails=0 +pass() { echo " PASS $*"; } +fail() { echo " FAIL $*"; fails=$((fails + 1)); } + +# Here-string, never `... | grep -q`: under pipefail, -q exits at the first +# match and SIGPIPEs the writer, sinking the pipeline, so a match reads as a +# failure -- but only above roughly 100K of input, which looks like a flake. +contains() { grep -q -- "$2" <<<"$1"; } + +# The envoy image ships neither curl nor wget, but it has bash, so /dev/tcp is +# the way in -- HTTP/1.1, because the admin listener answers 1.0 with 426. +admin_get() { # deploy loopback port path + ${K} -n "${NS}" exec "deploy/$1" -c envoy -- bash -c \ + "exec 3<>/dev/tcp/$2/$3; printf 'GET $4 HTTP/1.1\r\nHost: a\r\nConnection: close\r\n\r\n' >&3; cat <&3" \ + 2>/dev/null | tr -d '\r' +} + +echo "== cluster IP families ==" +# The Service families are the ground truth for what to assert. PreferDualStack +# resolves to whatever the cluster supports, so this reads back one entry on a +# single-stack cluster and two on dual-stack, in cluster-primary order. +families="$(${K} -n "${NS}" get svc atenet-router -o jsonpath='{.spec.ipFamilies}' \ + | tr -d '[]"' | tr ',' ' ')" +[ -n "${families}" ] || { echo "!! atenet-router Service not found in ${NS}"; exit 1; } +echo " cluster serves: ${families}" + +echo "== gateway pods are up, and stayed up ==" +# Restarts and readiness, not phase: a startup-probe refusal leaves the pod +# Running and kills it on a timer, so phase alone reads as healthy. +for app in atenet-router atenet-egress; do + # Every pod, not .items[0]: mid-rollout the old healthy pod and the new + # broken one coexist, and whichever sorts first would decide the verdict. + pods="$(${K} -n "${NS}" get pod -l "app=${app}" -o jsonpath='{.items[*].metadata.name}')" + [ -n "${pods}" ] || { fail "${app}: no pods"; continue; } + for pod in ${pods}; do + # A draining pod from the previous ReplicaSet is not a verdict on this one. + if [ -n "$(${K} -n "${NS}" get pod "${pod}" -o jsonpath='{.metadata.deletionTimestamp}')" ]; then + echo " .... ${pod}: terminating, skipped" + continue + fi + status="$(${K} -n "${NS}" get pod "${pod}" \ + -o jsonpath='{.status.phase} {range .status.containerStatuses[*]}{.name}={.ready}/{.restartCount} {end}')" + bad="$(grep -c false <<<"$(${K} -n "${NS}" get pod "${pod}" \ + -o jsonpath='{.status.containerStatuses[*].ready}')" || true)" + restarts="$(awk '{s+=$1} END {print s+0}' <<<"$(${K} -n "${NS}" get pod "${pod}" \ + -o jsonpath='{range .status.containerStatuses[*]}{.restartCount}{"\n"}{end}')")" + if [ "${bad}" -eq 0 ] && [ "${restarts}" -eq 0 ]; then + pass "${pod}: ${status}" + else + fail "${pod}: ${status}" + # The two lines that identify this failure: what Envoy bound, and what + # the kubelet dialed. Dumping the Envoy log instead buries both. + ${K} -n "${NS}" logs "${pod}" --all-containers --tail=-1 2>/dev/null \ + | grep "admin address" | sed 's/^/ envoy: /' || true + ${K} -n "${NS}" get event --field-selector "involvedObject.name=${pod}" \ + -o jsonpath='{range .items[?(@.reason=="Unhealthy")]}{.message}{"\n"}{end}' 2>/dev/null \ + | sort -u | head -3 | sed 's/^/ kubelet: /' || true + fi + done +done + +echo "== admin sockets bound on :: (from the Envoy startup log) ==" +for app in atenet-router:9901 atenet-egress:15000; do + a="${app%:*}"; port="${app#*:}" + # --all-containers: the router's Envoy is a child of the atenet-router + # process, so its startup log lands in that container, not in "envoy". + logs="$(${K} -n "${NS}" logs -l "app=${a}" --all-containers --tail=-1 2>/dev/null || true)" + if contains "${logs}" "admin address: \[::\]:${port}"; then + pass "${a}: admin address: [::]:${port}" + else + fail "${a}: no '[::]:${port}' admin line (v4-wildcard bind, or Envoy never started)" + fi +done + +echo "== IPv4 loopback still serves the admin socket ==" +# What ipv4_compat buys, and the only check that catches its loss: the router's +# health check (dataplane.go) and the egress drainer (--envoy-admin-address) +# dial 127.0.0.1, as literals in Go code that no manifest edit will follow. +for app in atenet-router:9901 atenet-egress:15000; do + a="${app%:*}"; port="${app#*:}" + for loop in 127.0.0.1 ::1; do + # || true: a refused connection is a result to report, not a reason to die. + line="$(admin_get "${a}" "${loop}" "${port}" /ready | head -1 || true)" + case "${line}" in + "HTTP/1.1 200 OK") pass "${a}: ${loop}:${port}/ready -> 200" ;; + "") fail "${a}: ${loop}:${port}/ready -> no answer (socket not bound for this family)" ;; + *) fail "${a}: ${loop}:${port}/ready -> ${line}" ;; + esac + done +done + +echo "== ipv4_compat, read from /config_dump ==" +# Never from /listeners: it reports resolved bound addresses and never emits +# the flag, so a check written against it passes on a manifest that lost it. +for app in atenet-router:9901:1 atenet-egress:15000:2; do + a="$(echo "${app}" | cut -d: -f1)" + port="$(echo "${app}" | cut -d: -f2)" + want="$(echo "${app}" | cut -d: -f3)" + got="$(grep -c '"ipv4_compat": true' <<<"$(admin_get "${a}" 127.0.0.1 "${port}" /config_dump)" || true)" + # Router: the admin socket only. Egress: admin plus the :443 listener, which + # Envoy reports twice (bootstrap and static views), so the floor is 2. + if [ "${got}" -ge "${want}" ]; then + pass "${a}: ${got} ipv4_compat socket(s) in /config_dump (want >= ${want})" + else + fail "${a}: ${got} ipv4_compat socket(s) in /config_dump, want >= ${want}" + fi +done + +echo "== ingress listeners keep an IPv4 primary and gain a :: address ==" +# The asymmetry is deliberate and easy to "clean up" into a regression: these +# are two sockets, so the v4 one stays as-is and ipv4_compat must be false on +# the v6 one. The admin sockets above are one socket, so they need the flag. +rdump="$(admin_get atenet-router 127.0.0.1 9901 /config_dump)" +for l in ingress_http_listener:8080 ingress_https_listener:8443; do + name="${l%:*}"; port="${l#*:}" + block="$(sed -n "/\"name\": \"${name}\"/,/last_updated/p" <<<"${rdump}")" + prim=0; addl=0 + contains "$(grep -A2 '"address": "0.0.0.0"' <<<"${block}" || true)" "\"port_value\": ${port}" && prim=1 + contains "$(sed -n '/additional_addresses/,/]/p' <<<"${block}")" '"address": "::"' && addl=1 + if [ "${prim}" = 1 ] && [ "${addl}" = 1 ]; then + pass "${name}: 0.0.0.0:${port} primary + :: additional" + else + fail "${name}: primary-0.0.0.0=${prim} additional-::=${addl}" + fi +done + +echo "== a pod can reach each gateway on every family the cluster has ==" +# Blocking delete: a --wait=false here races its own re-create and the run +# fails with AlreadyExists on any second invocation. +${K} -n "${NS}" delete pod "${PROBE_POD}" --ignore-not-found --timeout=60s >/dev/null 2>&1 || true +${K} -n "${NS}" run "${PROBE_POD}" --image="${PROBE_IMAGE}" --restart=Never \ + --command -- sleep 600 >/dev/null +trap '${K} -n "${NS}" delete pod "${PROBE_POD}" --ignore-not-found --wait=false >/dev/null 2>&1 || true' EXIT +${K} -n "${NS}" wait --for=condition=Ready "pod/${PROBE_POD}" --timeout=120s >/dev/null + +# One ClusterIP per family, in the same order as .spec.ipFamilies. +for app in atenet-router:80 atenet-egress:443; do + a="${app%:*}"; port="${app#*:}" + ips="$(${K} -n "${NS}" get svc "${a}" -o jsonpath='{.spec.clusterIPs}' | tr -d '[]"' | tr ',' ' ')" + i=0 + for ip in ${ips}; do + fam="$(echo "${families}" | cut -d' ' -f$((i + 1)))"; i=$((i + 1)) + # Bracket v6 literals for the URL; a bare colon would read as a port. + case "${ip}" in *:*) target="[${ip}]";; *) target="${ip}";; esac + # Connect, don't request: the router 404s without a Host header and egress + # :443 speaks mTLS CONNECT, so neither answers 200 when it is working. + if ${K} -n "${NS}" exec "${PROBE_POD}" -- \ + timeout 10 nc -z "${ip}" "${port}" >/dev/null 2>&1; then + pass "${a} ${fam} ${target}:${port} accepts connections" + else + fail "${a} ${fam} ${target}:${port} refused -- Service has the IP, nothing listens on it" + fi + done + [ "${i}" -eq "$(echo "${families}" | wc -w | tr -d ' ')" ] \ + || fail "${a}: ${i} ClusterIP(s) for $(echo "${families}" | wc -w | tr -d ' ') famil(ies) -- ipFamilyPolicy not PreferDualStack?" +done + +echo +if [ "${fails}" -eq 0 ]; then + echo "== PASS: both gateways bind and serve on ${families} ==" +else + echo "== FAIL: ${fails} check(s) failed ==" + exit 1 +fi diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index f413e5c95..b99376e7c 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" +IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" if [[ $# -gt 0 ]]; then case "$1" in @@ -31,6 +32,8 @@ if [[ $# -gt 0 ]]; then echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" + echo " (default: Google Public DNS). Override where those are unreachable." exit 0 ;; esac @@ -196,6 +199,55 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name} docker network connect "kind" "${reg_name}" fi +# 4.5. Give CoreDNS an IPv6 forwarder and a registry entry +# +# CoreDNS runs dnsPolicy: Default, inheriting the node's IPv4 resolver, which +# no pod here can reach, so external lookups SERVFAIL. Step 3's registry +# wiring is node-side, so it misses atelet too: that pull runs in atelet's own +# netns, where "kind-registry" NXDOMAINs. +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..." + reg_v6="$(docker inspect "${reg_name}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')" + if [[ -z "${reg_v6}" ]]; then + echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2 + exit 1 + fi + + corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}')" + # fallthrough is load-bearing: without it every name but the registry + # NXDOMAINs, cluster.local included -- one outage traded for a worse one. + search="forward . /etc/resolv.conf" + replace="hosts { + ${reg_v6} ${reg_name} + fallthrough + } + forward . ${IPV6_DNS_UPSTREAM}" + # $search unquoted: bash 3.2 splices the quotes in literally. The block's + # first line inherits the matched line's indentation. + patched="${corefile/$search/$replace}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " the Corefile layout changed upstream; update this block" >&2 + exit 1 + fi + + # A YAML patch file avoids escaping the Corefile's newlines into JSON. + { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ + > "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \ + --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \ + --timeout=120s + + # Its own script so it can be re-run against a live cluster: the hosts entry + # above is a snapshot of an address the registry can move off (#1049). + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" REG_NAME="${reg_name}" \ + IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM}" "${ROOT}"/hack/verify-ipv6-dns.sh +fi + # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." cat <&2 + exit 1 + ;; + esac +fi + +# Best-effort: only used to make the registry failure message actionable. +reg_v6="$(docker inspect "${REG_NAME}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' 2>/dev/null || true)" +reg_at="${reg_v6:+ at [${reg_v6}]:5000}" + +echo "Verifying DNS from a pod..." +# Probe from a pod, not the node: the node is dual-stack and passes either way. +# The registry leg fetches rather than resolves -- the hosts entry is AAAA-only, +# which fails nslookup's A query but satisfies getaddrinfo. +# +# --attach gives one stream and only the last leg's exit status, so each leg +# reports a marker on stdout and no failure message may contain one; PROBE_RAN +# separates a failed leg from a pod that never ran. Retry the pod, not the +# query: one that asks before CoreDNS settles stays broken for ~30s, while a +# fresh pod 10s later resolves first try. +probe="" +probe_max=4 +for ((probe_attempt = 1; probe_attempt <= probe_max; probe_attempt++)); do + attempt_out="$(kubectl --context="${KUBECTL_CONTEXT}" run "coredns-probe-$$-${probe_attempt}" \ + --rm --attach --quiet --restart=Never --image=busybox:1.36 --command -- \ + sh -c "echo PROBE_RAN + if out=\$(nslookup storage.googleapis.com 2>&1); then + echo RESOLVE_OK + else + echo \"resolve failed: \$(echo \"\$out\" | tail -2 | tr '\n' ' ')\" + fi + if out=\$(wget -T10 -O/dev/null http://${REG_NAME}:5000/v2/ 2>&1); then + echo REGISTRY_OK + else + echo \"registry fetch failed: \$(echo \"\$out\" | tail -1)\" + fi")" || true + # A pod that never started must not bury an earlier one's real failure. + if [[ "${attempt_out}" == *PROBE_RAN* ]]; then probe="${attempt_out}"; fi + # Only the resolve leg is a settling race; a down registry will not fix itself. + [[ "${probe}" == *RESOLVE_OK* ]] && break + if ((probe_attempt < probe_max)); then + echo " the cluster is not resolving yet; re-probing (attempt $((probe_attempt + 1)) of ${probe_max})..." + sleep 10 + fi +done +if [[ "${probe}" != *RESOLVE_OK* || "${probe}" != *REGISTRY_OK* ]]; then + if [[ "${probe}" != *PROBE_RAN* ]]; then + echo "error: the probe pod never ran, so CoreDNS is unverified" >&2 + echo " check that it scheduled and that 'busybox:1.36' pulled" >&2 + elif [[ "${probe}" != *RESOLVE_OK* ]]; then + echo "error: a pod cannot resolve an external name" >&2 + echo " IPV6_DNS_UPSTREAM is '${IPV6_DNS_UPSTREAM}'; set it to a reachable resolver" >&2 + else + echo "error: DNS works but a pod cannot reach '${REG_NAME}'${reg_at}" >&2 + echo " check the registry container is up and on the 'kind' network" >&2 + fi + if [[ -n "${probe}" ]]; then + echo " probe output was:" >&2 + printf '%s\n' "${probe}" | sed 's/^/ /' >&2 + fi + exit 1 +fi diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 591ef31fc..808a828e7 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -37,12 +37,13 @@ data: envoy.yaml: | admin: address: - socket_address: { address: 0.0.0.0, port_value: 15000 } + # ipv4_compat: the drainer dials this on IPv4 loopback (--envoy-admin-address, below). + socket_address: { address: "::", ipv4_compat: true, port_value: 15000 } static_resources: listeners: - name: egress address: - socket_address: { address: 0.0.0.0, port_value: 443 } + socket_address: { address: "::", ipv4_compat: true, port_value: 443 } filter_chains: # Named so ext_proc can read it back as xds.filter_chain_name. Must # match EgressFilterChainName in @@ -379,6 +380,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-egress ports: diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index e05e06efb..82d729ee5 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -86,7 +86,10 @@ data: admin: address: socket_address: - address: 0.0.0.0 + # ipv4_compat clears IPV6_V6ONLY, so this one socket serves both + # families; dataplane.go probes /ready over the IPv4 loopback. + address: "::" + ipv4_compat: true port_value: 9901 node: @@ -354,6 +357,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-router ports: