From a651cbc7f3a1428fc4d1eed61e9e78d8545a5970 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Sat, 8 Aug 2026 10:58:33 -0700 Subject: [PATCH 1/6] atenet/router: bind the Envoy ingress listeners dual-stack The HTTP and HTTPS ingress listeners bound 0.0.0.0 only, so on a dual-stack cluster Envoy answered on the router Service's IPv4 ClusterIP and on nothing at all for IPv6. Each primary socket now carries an additional "::" address on the same port. Ipv4Compat stays false on it: setting it would clear IPV6_V6ONLY and collide with the primary already bound to that port, and Envoy rejects the whole listener when an additional address fails to bind, taking down all ingress rather than the IPv6 half. IPv4-only clusters are unaffected -- the primary is untouched, and a host without IPv6 simply has no second socket to bind. First of three commits binding atenet's gateways dual-stack. --- cmd/atenet/internal/router/xds.go | 23 +++++++++++++++++ cmd/atenet/internal/router/xds_test.go | 35 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 1685b6262..56050e44d 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -1105,6 +1105,27 @@ func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing { } } +// dualStackAdditionalAddresses returns the IPv6 half of a dual-stack ingress +// listener, to pair with a primary 0.0.0.0 socket on the same port. Ipv4Compat +// stays false: clearing IPV6_V6ONLY would collide with that primary. +func dualStackAdditionalAddresses(port uint32) []*listenerv3.AdditionalAddress { + return []*listenerv3.AdditionalAddress{ + { + Address: &corev3.Address{ + Address: &corev3.Address_SocketAddress{ + SocketAddress: &corev3.SocketAddress{ + Address: "::", + Ipv4Compat: false, + PortSpecifier: &corev3.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + }, + } +} + func (x *XdsServer) buildListener() *listenerv3.Listener { hcm := x.buildHcm("ingress_http", true) @@ -1120,6 +1141,7 @@ func (x *XdsServer) buildListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.ingressPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ @@ -1179,6 +1201,7 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.httpsPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 8580685cb..e40ece3ad 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -158,6 +158,22 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { if sa.GetAddress() != "0.0.0.0" { t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) } + + addrs := l.GetAdditionalAddresses() + if len(addrs) == 0 { + t.Fatalf("Expected an additional address on %s, got none", IngressHTTPListener) + } + + asa := addrs[0].GetAddress().GetSocketAddress() + if asa.GetAddress() != "::" { + t.Errorf("Expected additional address '::', got %s", asa.GetAddress()) + } + if asa.GetIpv4Compat() { + t.Errorf("Expected additional address Ipv4Compat to be false") + } + if asa.GetPortValue() != 8081 { + t.Errorf("Expected additional port 8081, got %d", asa.GetPortValue()) + } } } @@ -196,6 +212,25 @@ func TestXdsServer_UpdateSnapshot_WithHttps(t *testing.T) { if sa.GetPortValue() != 8443 { t.Errorf("Expected port 8443, got %d", sa.GetPortValue()) } + if sa.GetAddress() != "0.0.0.0" { + t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) + } + + addrs := l.GetAdditionalAddresses() + if len(addrs) == 0 { + t.Fatalf("Expected an additional address on %s, got none", IngressHTTPSListener) + } + + asa := addrs[0].GetAddress().GetSocketAddress() + if asa.GetAddress() != "::" { + t.Errorf("Expected additional address '::', got %s", asa.GetAddress()) + } + if asa.GetIpv4Compat() { + t.Errorf("Expected additional address Ipv4Compat to be false") + } + if asa.GetPortValue() != 8443 { + t.Errorf("Expected additional port 8443, got %d", asa.GetPortValue()) + } // Verify the TLS config references the serving cert via SDS rather // than embedding it: inline filename DataSources are read only once From 9e1f8e0f170c2a1271aaad8e7df20df73a690bdb Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 12 Aug 2026 21:41:31 -0700 Subject: [PATCH 2/6] atenet/router: bind the admin socket and Service dual-stack The Envoy admin socket bound 0.0.0.0, and the atenet-router Service carried no ipFamilyPolicy -- which the API server defaults to SingleStack, one IPv4 ClusterIP and nothing else. Between them the router had no IPv6 address to answer on. The socket now binds "::" with ipv4_compat, one socket for both families, and the Service asks for PreferDualStack. ipv4_compat is load-bearing rather than incidental: dataplane.go health-checks the admin listener over http://127.0.0.1:9901/ready, so a bare "::" would report the dataplane component of /statusz unhealthy. Prefer, not Require, keeps the Service valid on a single-stack cluster, where it is a no-op; spec.ipFamilies is left alone because the primary family is immutable and the API server appends the secondary itself. --- manifests/ate-install/atenet-router.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index e05e06efb..82d729ee5 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -86,7 +86,10 @@ data: admin: address: socket_address: - address: 0.0.0.0 + # ipv4_compat clears IPV6_V6ONLY, so this one socket serves both + # families; dataplane.go probes /ready over the IPv4 loopback. + address: "::" + ipv4_compat: true port_value: 9901 node: @@ -354,6 +357,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-router ports: From b49cebea388c85832ae0d34b7e0f7e4c023a63b7 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 13 Aug 2026 07:20:18 -0700 Subject: [PATCH 3/6] atenet/egress: bind the Envoy sockets and Service dual-stack The gateway's admin and :443 sockets bound 0.0.0.0, so on an IPv6-primary cluster the kubelet probed the pod on its only address and atenet-egress crashlooped -- Envoy started fine and logged "admin address: 0.0.0.0:15000" -- while an actor's CONNECT had no v6 path in. Both sockets now bind "::" with ipv4_compat, and the Service asks for PreferDualStack so a dual-stack cluster hands out an IPv6 ClusterIP to reach them on. ipv4_compat matters on the admin socket in particular: the ext-proc sidecar's drainer dials 127.0.0.1:15000, and envoydrain.go reads a refusal there as "Envoy already exited" and skips the drain silently. --- manifests/ate-install/atenet-egress.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 591ef31fc..808a828e7 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -37,12 +37,13 @@ data: envoy.yaml: | admin: address: - socket_address: { address: 0.0.0.0, port_value: 15000 } + # ipv4_compat: the drainer dials this on IPv4 loopback (--envoy-admin-address, below). + socket_address: { address: "::", ipv4_compat: true, port_value: 15000 } static_resources: listeners: - name: egress address: - socket_address: { address: 0.0.0.0, port_value: 443 } + socket_address: { address: "::", ipv4_compat: true, port_value: 443 } filter_chains: # Named so ext_proc can read it back as xds.filter_chain_name. Must # match EgressFilterChainName in @@ -379,6 +380,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-egress ports: From 6d6f12354e6ad07ea8eecd7530b01490d48324c8 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 19 Aug 2026 08:49:40 -0700 Subject: [PATCH 4/6] hack: fix DNS on IPv6-only kind clusters On a fresh IP_FAMILY=ipv6 cluster nothing resolves from inside a pod and no actor boots: CoreDNS inherits the node's IPv4 resolver, which a v6-only pod cannot reach. Two Corefile clauses fix it -- a hosts entry mapping kind-registry to its IPv6 address, and a forward to an IPv6 upstream, overridable with IPV6_DNS_UPSTREAM. IPv4 and dual-stack clusters are unchanged, and atenet-egress still crashloops on v6-only for an unrelated Envoy bind bug. Asking once was not enough to prove that: about half of fresh clusters do not answer the first query, and a pod that goes unanswered stays unanswered, so the check re-asks with a new pod and prints what the pod saw when it gives up. It lives in hack/verify-ipv6-dns.sh rather than inline, because the hosts entry records an address the registry can move off and there was no way to re-check a cluster without rebuilding it. --- hack/create-kind-cluster.sh | 52 ++++++++++++++++++ hack/verify-ipv6-dns.sh | 106 ++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100755 hack/verify-ipv6-dns.sh diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index f413e5c95..b99376e7c 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" +IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" if [[ $# -gt 0 ]]; then case "$1" in @@ -31,6 +32,8 @@ if [[ $# -gt 0 ]]; then echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" + echo " (default: Google Public DNS). Override where those are unreachable." exit 0 ;; esac @@ -196,6 +199,55 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name} docker network connect "kind" "${reg_name}" fi +# 4.5. Give CoreDNS an IPv6 forwarder and a registry entry +# +# CoreDNS runs dnsPolicy: Default, inheriting the node's IPv4 resolver, which +# no pod here can reach, so external lookups SERVFAIL. Step 3's registry +# wiring is node-side, so it misses atelet too: that pull runs in atelet's own +# netns, where "kind-registry" NXDOMAINs. +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..." + reg_v6="$(docker inspect "${reg_name}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')" + if [[ -z "${reg_v6}" ]]; then + echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2 + exit 1 + fi + + corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}')" + # fallthrough is load-bearing: without it every name but the registry + # NXDOMAINs, cluster.local included -- one outage traded for a worse one. + search="forward . /etc/resolv.conf" + replace="hosts { + ${reg_v6} ${reg_name} + fallthrough + } + forward . ${IPV6_DNS_UPSTREAM}" + # $search unquoted: bash 3.2 splices the quotes in literally. The block's + # first line inherits the matched line's indentation. + patched="${corefile/$search/$replace}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " the Corefile layout changed upstream; update this block" >&2 + exit 1 + fi + + # A YAML patch file avoids escaping the Corefile's newlines into JSON. + { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ + > "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \ + --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \ + --timeout=120s + + # Its own script so it can be re-run against a live cluster: the hosts entry + # above is a snapshot of an address the registry can move off (#1049). + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" REG_NAME="${reg_name}" \ + IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM}" "${ROOT}"/hack/verify-ipv6-dns.sh +fi + # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." cat <&2 + exit 1 + ;; + esac +fi + +# Best-effort: only used to make the registry failure message actionable. +reg_v6="$(docker inspect "${REG_NAME}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' 2>/dev/null || true)" +reg_at="${reg_v6:+ at [${reg_v6}]:5000}" + +echo "Verifying DNS from a pod..." +# Probe from a pod, not the node: the node is dual-stack and passes either way. +# The registry leg fetches rather than resolves -- the hosts entry is AAAA-only, +# which fails nslookup's A query but satisfies getaddrinfo. +# +# --attach gives one stream and only the last leg's exit status, so each leg +# reports a marker on stdout and no failure message may contain one; PROBE_RAN +# separates a failed leg from a pod that never ran. Retry the pod, not the +# query: one that asks before CoreDNS settles stays broken for ~30s, while a +# fresh pod 10s later resolves first try. +probe="" +probe_max=4 +for ((probe_attempt = 1; probe_attempt <= probe_max; probe_attempt++)); do + attempt_out="$(kubectl --context="${KUBECTL_CONTEXT}" run "coredns-probe-$$-${probe_attempt}" \ + --rm --attach --quiet --restart=Never --image=busybox:1.36 --command -- \ + sh -c "echo PROBE_RAN + if out=\$(nslookup storage.googleapis.com 2>&1); then + echo RESOLVE_OK + else + echo \"resolve failed: \$(echo \"\$out\" | tail -2 | tr '\n' ' ')\" + fi + if out=\$(wget -T10 -O/dev/null http://${REG_NAME}:5000/v2/ 2>&1); then + echo REGISTRY_OK + else + echo \"registry fetch failed: \$(echo \"\$out\" | tail -1)\" + fi")" || true + # A pod that never started must not bury an earlier one's real failure. + if [[ "${attempt_out}" == *PROBE_RAN* ]]; then probe="${attempt_out}"; fi + # Only the resolve leg is a settling race; a down registry will not fix itself. + [[ "${probe}" == *RESOLVE_OK* ]] && break + if ((probe_attempt < probe_max)); then + echo " the cluster is not resolving yet; re-probing (attempt $((probe_attempt + 1)) of ${probe_max})..." + sleep 10 + fi +done +if [[ "${probe}" != *RESOLVE_OK* || "${probe}" != *REGISTRY_OK* ]]; then + if [[ "${probe}" != *PROBE_RAN* ]]; then + echo "error: the probe pod never ran, so CoreDNS is unverified" >&2 + echo " check that it scheduled and that 'busybox:1.36' pulled" >&2 + elif [[ "${probe}" != *RESOLVE_OK* ]]; then + echo "error: a pod cannot resolve an external name" >&2 + echo " IPV6_DNS_UPSTREAM is '${IPV6_DNS_UPSTREAM}'; set it to a reachable resolver" >&2 + else + echo "error: DNS works but a pod cannot reach '${REG_NAME}'${reg_at}" >&2 + echo " check the registry container is up and on the 'kind' network" >&2 + fi + if [[ -n "${probe}" ]]; then + echo " probe output was:" >&2 + printf '%s\n' "${probe}" | sed 's/^/ /' >&2 + fi + exit 1 +fi From 84fc597c88ec580814258c295367c05e1903868a Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Tue, 18 Aug 2026 16:02:58 -0700 Subject: [PATCH 5/6] hack: check the atenet gateways on every IP family Whether the router and egress gateways really serve on every family the cluster has was only ever checked by hand, and the checks are easy to get wrong -- Envoy's /listeners reports resolved addresses and never emits ipv4_compat, so an assertion written against it passes on a manifest that lost the flag. This script reads the families off the Service and reports every check it makes, so one run is valid on ipv4, dual, and ipv6. Worth running on all three, because they fail differently. A v4-wildcard bind is a silent data-path gap on dual-stack, where the Service has an IPv6 ClusterIP nothing listens on, and a crashloop on IPv6-only, where the kubelet probes the pod on its only address. --- hack/check-atenet-ip-families.sh | 199 +++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100755 hack/check-atenet-ip-families.sh diff --git a/hack/check-atenet-ip-families.sh b/hack/check-atenet-ip-families.sh new file mode 100755 index 000000000..569d6b680 --- /dev/null +++ b/hack/check-atenet-ip-families.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Preconditions: +# IP_FAMILY=ipv4|dual|ipv6 hack/create-kind-cluster.sh +# hack/install-ate-kind.sh --deploy-ate-system +# +# Checks that both atenet gateways bind and serve on every IP family the +# cluster has. The assertions are read off the cluster, so one run is valid on +# all three families -- worth doing on each, because they fail differently: on +# dual-stack a v4-wildcard bind is a silent data-path gap (the Service has an +# IPv6 ClusterIP nothing listens on), on IPv6-only it is a crashloop, because +# the kubelet probes the pod on its only address. +set -o errexit -o nounset -o pipefail +ROOT="$(git rev-parse --show-toplevel)"; cd "${ROOT}" + +CTX="${KUBECTL_CONTEXT:-kind-kind}" +K="kubectl --context ${CTX}" +NS="${NS:-ate-system}" +PROBE_IMAGE="${PROBE_IMAGE:-busybox:1.36}" +PROBE_POD="atenet-ipfamily-probe" + +fails=0 +pass() { echo " PASS $*"; } +fail() { echo " FAIL $*"; fails=$((fails + 1)); } + +# Here-string, never `... | grep -q`: under pipefail, -q exits at the first +# match and SIGPIPEs the writer, sinking the pipeline, so a match reads as a +# failure -- but only above roughly 100K of input, which looks like a flake. +contains() { grep -q -- "$2" <<<"$1"; } + +# The envoy image ships neither curl nor wget, but it has bash, so /dev/tcp is +# the way in -- HTTP/1.1, because the admin listener answers 1.0 with 426. +admin_get() { # deploy loopback port path + ${K} -n "${NS}" exec "deploy/$1" -c envoy -- bash -c \ + "exec 3<>/dev/tcp/$2/$3; printf 'GET $4 HTTP/1.1\r\nHost: a\r\nConnection: close\r\n\r\n' >&3; cat <&3" \ + 2>/dev/null | tr -d '\r' +} + +echo "== cluster IP families ==" +# The Service families are the ground truth for what to assert. PreferDualStack +# resolves to whatever the cluster supports, so this reads back one entry on a +# single-stack cluster and two on dual-stack, in cluster-primary order. +families="$(${K} -n "${NS}" get svc atenet-router -o jsonpath='{.spec.ipFamilies}' \ + | tr -d '[]"' | tr ',' ' ')" +[ -n "${families}" ] || { echo "!! atenet-router Service not found in ${NS}"; exit 1; } +echo " cluster serves: ${families}" + +echo "== gateway pods are up, and stayed up ==" +# Restarts and readiness, not phase: a startup-probe refusal leaves the pod +# Running and kills it on a timer, so phase alone reads as healthy. +for app in atenet-router atenet-egress; do + # Every pod, not .items[0]: mid-rollout the old healthy pod and the new + # broken one coexist, and whichever sorts first would decide the verdict. + pods="$(${K} -n "${NS}" get pod -l "app=${app}" -o jsonpath='{.items[*].metadata.name}')" + [ -n "${pods}" ] || { fail "${app}: no pods"; continue; } + for pod in ${pods}; do + # A draining pod from the previous ReplicaSet is not a verdict on this one. + if [ -n "$(${K} -n "${NS}" get pod "${pod}" -o jsonpath='{.metadata.deletionTimestamp}')" ]; then + echo " .... ${pod}: terminating, skipped" + continue + fi + status="$(${K} -n "${NS}" get pod "${pod}" \ + -o jsonpath='{.status.phase} {range .status.containerStatuses[*]}{.name}={.ready}/{.restartCount} {end}')" + bad="$(grep -c false <<<"$(${K} -n "${NS}" get pod "${pod}" \ + -o jsonpath='{.status.containerStatuses[*].ready}')" || true)" + restarts="$(awk '{s+=$1} END {print s+0}' <<<"$(${K} -n "${NS}" get pod "${pod}" \ + -o jsonpath='{range .status.containerStatuses[*]}{.restartCount}{"\n"}{end}')")" + if [ "${bad}" -eq 0 ] && [ "${restarts}" -eq 0 ]; then + pass "${pod}: ${status}" + else + fail "${pod}: ${status}" + # The two lines that identify this failure: what Envoy bound, and what + # the kubelet dialed. Dumping the Envoy log instead buries both. + ${K} -n "${NS}" logs "${pod}" --all-containers --tail=-1 2>/dev/null \ + | grep "admin address" | sed 's/^/ envoy: /' || true + ${K} -n "${NS}" get event --field-selector "involvedObject.name=${pod}" \ + -o jsonpath='{range .items[?(@.reason=="Unhealthy")]}{.message}{"\n"}{end}' 2>/dev/null \ + | sort -u | head -3 | sed 's/^/ kubelet: /' || true + fi + done +done + +echo "== admin sockets bound on :: (from the Envoy startup log) ==" +for app in atenet-router:9901 atenet-egress:15000; do + a="${app%:*}"; port="${app#*:}" + # --all-containers: the router's Envoy is a child of the atenet-router + # process, so its startup log lands in that container, not in "envoy". + logs="$(${K} -n "${NS}" logs -l "app=${a}" --all-containers --tail=-1 2>/dev/null || true)" + if contains "${logs}" "admin address: \[::\]:${port}"; then + pass "${a}: admin address: [::]:${port}" + else + fail "${a}: no '[::]:${port}' admin line (v4-wildcard bind, or Envoy never started)" + fi +done + +echo "== IPv4 loopback still serves the admin socket ==" +# What ipv4_compat buys, and the only check that catches its loss: the router's +# health check (dataplane.go) and the egress drainer (--envoy-admin-address) +# dial 127.0.0.1, as literals in Go code that no manifest edit will follow. +for app in atenet-router:9901 atenet-egress:15000; do + a="${app%:*}"; port="${app#*:}" + for loop in 127.0.0.1 ::1; do + # || true: a refused connection is a result to report, not a reason to die. + line="$(admin_get "${a}" "${loop}" "${port}" /ready | head -1 || true)" + case "${line}" in + "HTTP/1.1 200 OK") pass "${a}: ${loop}:${port}/ready -> 200" ;; + "") fail "${a}: ${loop}:${port}/ready -> no answer (socket not bound for this family)" ;; + *) fail "${a}: ${loop}:${port}/ready -> ${line}" ;; + esac + done +done + +echo "== ipv4_compat, read from /config_dump ==" +# Never from /listeners: it reports resolved bound addresses and never emits +# the flag, so a check written against it passes on a manifest that lost it. +for app in atenet-router:9901:1 atenet-egress:15000:2; do + a="$(echo "${app}" | cut -d: -f1)" + port="$(echo "${app}" | cut -d: -f2)" + want="$(echo "${app}" | cut -d: -f3)" + got="$(grep -c '"ipv4_compat": true' <<<"$(admin_get "${a}" 127.0.0.1 "${port}" /config_dump)" || true)" + # Router: the admin socket only. Egress: admin plus the :443 listener, which + # Envoy reports twice (bootstrap and static views), so the floor is 2. + if [ "${got}" -ge "${want}" ]; then + pass "${a}: ${got} ipv4_compat socket(s) in /config_dump (want >= ${want})" + else + fail "${a}: ${got} ipv4_compat socket(s) in /config_dump, want >= ${want}" + fi +done + +echo "== ingress listeners keep an IPv4 primary and gain a :: address ==" +# The asymmetry is deliberate and easy to "clean up" into a regression: these +# are two sockets, so the v4 one stays as-is and ipv4_compat must be false on +# the v6 one. The admin sockets above are one socket, so they need the flag. +rdump="$(admin_get atenet-router 127.0.0.1 9901 /config_dump)" +for l in ingress_http_listener:8080 ingress_https_listener:8443; do + name="${l%:*}"; port="${l#*:}" + block="$(sed -n "/\"name\": \"${name}\"/,/last_updated/p" <<<"${rdump}")" + prim=0; addl=0 + contains "$(grep -A2 '"address": "0.0.0.0"' <<<"${block}" || true)" "\"port_value\": ${port}" && prim=1 + contains "$(sed -n '/additional_addresses/,/]/p' <<<"${block}")" '"address": "::"' && addl=1 + if [ "${prim}" = 1 ] && [ "${addl}" = 1 ]; then + pass "${name}: 0.0.0.0:${port} primary + :: additional" + else + fail "${name}: primary-0.0.0.0=${prim} additional-::=${addl}" + fi +done + +echo "== a pod can reach each gateway on every family the cluster has ==" +# Blocking delete: a --wait=false here races its own re-create and the run +# fails with AlreadyExists on any second invocation. +${K} -n "${NS}" delete pod "${PROBE_POD}" --ignore-not-found --timeout=60s >/dev/null 2>&1 || true +${K} -n "${NS}" run "${PROBE_POD}" --image="${PROBE_IMAGE}" --restart=Never \ + --command -- sleep 600 >/dev/null +trap '${K} -n "${NS}" delete pod "${PROBE_POD}" --ignore-not-found --wait=false >/dev/null 2>&1 || true' EXIT +${K} -n "${NS}" wait --for=condition=Ready "pod/${PROBE_POD}" --timeout=120s >/dev/null + +# One ClusterIP per family, in the same order as .spec.ipFamilies. +for app in atenet-router:80 atenet-egress:443; do + a="${app%:*}"; port="${app#*:}" + ips="$(${K} -n "${NS}" get svc "${a}" -o jsonpath='{.spec.clusterIPs}' | tr -d '[]"' | tr ',' ' ')" + i=0 + for ip in ${ips}; do + fam="$(echo "${families}" | cut -d' ' -f$((i + 1)))"; i=$((i + 1)) + # Bracket v6 literals for the URL; a bare colon would read as a port. + case "${ip}" in *:*) target="[${ip}]";; *) target="${ip}";; esac + # Connect, don't request: the router 404s without a Host header and egress + # :443 speaks mTLS CONNECT, so neither answers 200 when it is working. + if ${K} -n "${NS}" exec "${PROBE_POD}" -- \ + timeout 10 nc -z "${ip}" "${port}" >/dev/null 2>&1; then + pass "${a} ${fam} ${target}:${port} accepts connections" + else + fail "${a} ${fam} ${target}:${port} refused -- Service has the IP, nothing listens on it" + fi + done + [ "${i}" -eq "$(echo "${families}" | wc -w | tr -d ' ')" ] \ + || fail "${a}: ${i} ClusterIP(s) for $(echo "${families}" | wc -w | tr -d ' ') famil(ies) -- ipFamilyPolicy not PreferDualStack?" +done + +echo +if [ "${fails}" -eq 0 ]; then + echo "== PASS: both gateways bind and serve on ${families} ==" +else + echo "== FAIL: ${fails} check(s) failed ==" + exit 1 +fi From 6e2c529bb58d07c5146c681c25cfd092c846b4bb Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 20 Aug 2026 08:41:09 -0700 Subject: [PATCH 6/6] hack: read the IP-family checks off the current generation only The admin-socket check used `kubectl logs -l app=...`, which concatenates the logs of every pod carrying the label -- including the previous generation still draining after a rollout. A substring match over that buffer passes on the very config that was just backed out: on a dual-stack cluster the check reported `[::]:9901` green while the only live router had logged `admin address: 0.0.0.0:9901`. `admin_get` had the same flaw via `exec deploy/`, which resolves to whichever pod kubectl happens to pick. Both now select pods by label and skip any carrying a deletionTimestamp, as the pod-health loop above them already did, and the admin check requires every current pod to show the line instead of any one of them. --- hack/check-atenet-ip-families.sh | 46 +++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/hack/check-atenet-ip-families.sh b/hack/check-atenet-ip-families.sh index 569d6b680..af72e0be8 100755 --- a/hack/check-atenet-ip-families.sh +++ b/hack/check-atenet-ip-families.sh @@ -42,10 +42,28 @@ fail() { echo " FAIL $*"; fails=$((fails + 1)); } # failure -- but only above roughly 100K of input, which looks like a flake. contains() { grep -q -- "$2" <<<"$1"; } +# Pods of the current generation only. A pod with a deletionTimestamp is still +# listed, still serving, and still carries the *previous* config -- so anything +# selected by label alone can be answered by the change we just backed out. +current_pods() { # app + local pod + for pod in $(${K} -n "${NS}" get pod -l "app=$1" -o jsonpath='{.items[*].metadata.name}'); do + [ -n "$(${K} -n "${NS}" get pod "${pod}" -o jsonpath='{.metadata.deletionTimestamp}')" ] && continue + echo "${pod}" + done +} + # The envoy image ships neither curl nor wget, but it has bash, so /dev/tcp is # the way in -- HTTP/1.1, because the admin listener answers 1.0 with 426. -admin_get() { # deploy loopback port path - ${K} -n "${NS}" exec "deploy/$1" -c envoy -- bash -c \ +# Exec into a named current pod, never deploy/: that resolves to whatever +# pod kubectl picks, which mid-rollout may be the draining one. +admin_get() { # app loopback port path + local pod + # Not `| head -1`: head exits at the first line, SIGPIPEs current_pods on the + # second, and under pipefail that sinks the assignment and trips errexit. + pod="$(current_pods "$1")"; pod="${pod%%$'\n'*}" + [ -n "${pod}" ] || return 0 + ${K} -n "${NS}" exec "${pod}" -c envoy -- bash -c \ "exec 3<>/dev/tcp/$2/$3; printf 'GET $4 HTTP/1.1\r\nHost: a\r\nConnection: close\r\n\r\n' >&3; cat <&3" \ 2>/dev/null | tr -d '\r' } @@ -97,14 +115,22 @@ done echo "== admin sockets bound on :: (from the Envoy startup log) ==" for app in atenet-router:9901 atenet-egress:15000; do a="${app%:*}"; port="${app#*:}" - # --all-containers: the router's Envoy is a child of the atenet-router - # process, so its startup log lands in that container, not in "envoy". - logs="$(${K} -n "${NS}" logs -l "app=${a}" --all-containers --tail=-1 2>/dev/null || true)" - if contains "${logs}" "admin address: \[::\]:${port}"; then - pass "${a}: admin address: [::]:${port}" - else - fail "${a}: no '[::]:${port}' admin line (v4-wildcard bind, or Envoy never started)" - fi + pods="$(current_pods "${a}")" + [ -n "${pods}" ] || { fail "${a}: no pods of the current generation"; continue; } + # Per pod, and every one has to say it. `logs -l app=` concatenates the + # draining previous generation's log into the same buffer, so a substring + # match there passes on the config we just backed out. + for pod in ${pods}; do + # --all-containers: the router's Envoy is a child of the atenet-router + # process, so its startup log lands in that container, not in "envoy". + logs="$(${K} -n "${NS}" logs "${pod}" --all-containers --tail=-1 2>/dev/null || true)" + if contains "${logs}" "admin address: \[::\]:${port}"; then + pass "${pod}: admin address: [::]:${port}" + else + fail "${pod}: no '[::]:${port}' admin line (v4-wildcard bind, or Envoy never started)" + grep -o 'admin address: [^ ]*' <<<"${logs}" | sort -u | sed 's/^/ envoy: /' || true + fi + done done echo "== IPv4 loopback still serves the admin socket =="