[do not merge] hack: check the atenet gateways on every IP family

cmd/atenet/internal/router/xds.go

@@ -1105,6 +1105,27 @@ func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing {
 	}
 }
 
+// dualStackAdditionalAddresses returns the IPv6 half of a dual-stack ingress
+// listener, to pair with a primary 0.0.0.0 socket on the same port. Ipv4Compat
+// stays false: clearing IPV6_V6ONLY would collide with that primary.
+func dualStackAdditionalAddresses(port uint32) []*listenerv3.AdditionalAddress {
+	return []*listenerv3.AdditionalAddress{
+		{
+			Address: &corev3.Address{
+				Address: &corev3.Address_SocketAddress{
+					SocketAddress: &corev3.SocketAddress{
+						Address:    "::",
+						Ipv4Compat: false,
+						PortSpecifier: &corev3.SocketAddress_PortValue{
+							PortValue: port,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func (x *XdsServer) buildListener() *listenerv3.Listener {
 	hcm := x.buildHcm("ingress_http", true)
 
@@ -1120,6 +1141,7 @@ func (x *XdsServer) buildListener() *listenerv3.Listener {
 				},
 			},
 		},
+		AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.ingressPort)),
 		FilterChains: []*listenerv3.FilterChain{
 			{
 				Filters: []*listenerv3.Filter{
@@ -1179,6 +1201,7 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener {
 				},
 			},
 		},
+		AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.httpsPort)),
 		FilterChains: []*listenerv3.FilterChain{
 			{
 				Filters: []*listenerv3.Filter{

cmd/atenet/internal/router/xds_test.go

@@ -158,6 +158,22 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) {
 		if sa.GetAddress() != "0.0.0.0" {
 			t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress())
 		}
+
+		addrs := l.GetAdditionalAddresses()
+		if len(addrs) == 0 {
+			t.Fatalf("Expected an additional address on %s, got none", IngressHTTPListener)
+		}
+
+		asa := addrs[0].GetAddress().GetSocketAddress()
+		if asa.GetAddress() != "::" {
+			t.Errorf("Expected additional address '::', got %s", asa.GetAddress())
+		}
+		if asa.GetIpv4Compat() {
+			t.Errorf("Expected additional address Ipv4Compat to be false")
+		}
+		if asa.GetPortValue() != 8081 {
+			t.Errorf("Expected additional port 8081, got %d", asa.GetPortValue())
+		}
 	}
 }
 
@@ -196,6 +212,25 @@ func TestXdsServer_UpdateSnapshot_WithHttps(t *testing.T) {
 		if sa.GetPortValue() != 8443 {
 			t.Errorf("Expected port 8443, got %d", sa.GetPortValue())
 		}
+		if sa.GetAddress() != "0.0.0.0" {
+			t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress())
+		}
+
+		addrs := l.GetAdditionalAddresses()
+		if len(addrs) == 0 {
+			t.Fatalf("Expected an additional address on %s, got none", IngressHTTPSListener)
+		}
+
+		asa := addrs[0].GetAddress().GetSocketAddress()
+		if asa.GetAddress() != "::" {
+			t.Errorf("Expected additional address '::', got %s", asa.GetAddress())
+		}
+		if asa.GetIpv4Compat() {
+			t.Errorf("Expected additional address Ipv4Compat to be false")
+		}
+		if asa.GetPortValue() != 8443 {
+			t.Errorf("Expected additional port 8443, got %d", asa.GetPortValue())
+		}
 
 		// Verify the TLS config references the serving cert via SDS rather
 		// than embedding it: inline filename DataSources are read only once

hack/check-atenet-ip-families.sh

@@ -0,0 +1,199 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Preconditions:
+#   IP_FAMILY=ipv4|dual|ipv6 hack/create-kind-cluster.sh
+#   hack/install-ate-kind.sh --deploy-ate-system
+#
+# Checks that both atenet gateways bind and serve on every IP family the
+# cluster has. The assertions are read off the cluster, so one run is valid on
+# all three families -- worth doing on each, because they fail differently: on
+# dual-stack a v4-wildcard bind is a silent data-path gap (the Service has an
+# IPv6 ClusterIP nothing listens on), on IPv6-only it is a crashloop, because
+# the kubelet probes the pod on its only address.
+set -o errexit -o nounset -o pipefail
+ROOT="$(git rev-parse --show-toplevel)"; cd "${ROOT}"
+
+CTX="${KUBECTL_CONTEXT:-kind-kind}"
+K="kubectl --context ${CTX}"
+NS="${NS:-ate-system}"
+PROBE_IMAGE="${PROBE_IMAGE:-busybox:1.36}"
+PROBE_POD="atenet-ipfamily-probe"
+
+fails=0
+pass() { echo "  PASS  $*"; }
+fail() { echo "  FAIL  $*"; fails=$((fails + 1)); }
+
+# Here-string, never `... | grep -q`: under pipefail, -q exits at the first
+# match and SIGPIPEs the writer, sinking the pipeline, so a match reads as a
+# failure -- but only above roughly 100K of input, which looks like a flake.
+contains() { grep -q -- "$2" <<<"$1"; }
+
+# The envoy image ships neither curl nor wget, but it has bash, so /dev/tcp is
+# the way in -- HTTP/1.1, because the admin listener answers 1.0 with 426.
+admin_get() { # deploy loopback port path
+  ${K} -n "${NS}" exec "deploy/$1" -c envoy -- bash -c \
+    "exec 3<>/dev/tcp/$2/$3; printf 'GET $4 HTTP/1.1\r\nHost: a\r\nConnection: close\r\n\r\n' >&3; cat <&3" \
+    2>/dev/null | tr -d '\r'
+}
+
+echo "== cluster IP families =="
+# The Service families are the ground truth for what to assert. PreferDualStack
+# resolves to whatever the cluster supports, so this reads back one entry on a
+# single-stack cluster and two on dual-stack, in cluster-primary order.
+families="$(${K} -n "${NS}" get svc atenet-router -o jsonpath='{.spec.ipFamilies}' \
+  | tr -d '[]"' | tr ',' ' ')"
+[ -n "${families}" ] || { echo "!! atenet-router Service not found in ${NS}"; exit 1; }
+echo "  cluster serves: ${families}"
+
+echo "== gateway pods are up, and stayed up =="
+# Restarts and readiness, not phase: a startup-probe refusal leaves the pod
+# Running and kills it on a timer, so phase alone reads as healthy.
+for app in atenet-router atenet-egress; do
+  # Every pod, not .items[0]: mid-rollout the old healthy pod and the new
+  # broken one coexist, and whichever sorts first would decide the verdict.
+  pods="$(${K} -n "${NS}" get pod -l "app=${app}" -o jsonpath='{.items[*].metadata.name}')"
+  [ -n "${pods}" ] || { fail "${app}: no pods"; continue; }
+  for pod in ${pods}; do
+    # A draining pod from the previous ReplicaSet is not a verdict on this one.
+    if [ -n "$(${K} -n "${NS}" get pod "${pod}" -o jsonpath='{.metadata.deletionTimestamp}')" ]; then
+      echo "  ....  ${pod}: terminating, skipped"
+      continue
+    fi
+    status="$(${K} -n "${NS}" get pod "${pod}" \
+      -o jsonpath='{.status.phase} {range .status.containerStatuses[*]}{.name}={.ready}/{.restartCount} {end}')"
+    bad="$(grep -c false <<<"$(${K} -n "${NS}" get pod "${pod}" \
+      -o jsonpath='{.status.containerStatuses[*].ready}')" || true)"
+    restarts="$(awk '{s+=$1} END {print s+0}' <<<"$(${K} -n "${NS}" get pod "${pod}" \
+      -o jsonpath='{range .status.containerStatuses[*]}{.restartCount}{"\n"}{end}')")"
+    if [ "${bad}" -eq 0 ] && [ "${restarts}" -eq 0 ]; then
+      pass "${pod}: ${status}"
+    else
+      fail "${pod}: ${status}"
+      # The two lines that identify this failure: what Envoy bound, and what
+      # the kubelet dialed. Dumping the Envoy log instead buries both.
+      ${K} -n "${NS}" logs "${pod}" --all-containers --tail=-1 2>/dev/null \
+        | grep "admin address" | sed 's/^/        envoy: /' || true
+      ${K} -n "${NS}" get event --field-selector "involvedObject.name=${pod}" \
+        -o jsonpath='{range .items[?(@.reason=="Unhealthy")]}{.message}{"\n"}{end}' 2>/dev/null \
+        | sort -u | head -3 | sed 's/^/        kubelet: /' || true
+    fi
+  done
+done
+
+echo "== admin sockets bound on :: (from the Envoy startup log) =="
+for app in atenet-router:9901 atenet-egress:15000; do
+  a="${app%:*}"; port="${app#*:}"
+  # --all-containers: the router's Envoy is a child of the atenet-router
+  # process, so its startup log lands in that container, not in "envoy".
+  logs="$(${K} -n "${NS}" logs -l "app=${a}" --all-containers --tail=-1 2>/dev/null || true)"
+  if contains "${logs}" "admin address: \[::\]:${port}"; then
+    pass "${a}: admin address: [::]:${port}"
+  else
+    fail "${a}: no '[::]:${port}' admin line (v4-wildcard bind, or Envoy never started)"
+  fi
+done
+
+echo "== IPv4 loopback still serves the admin socket =="
+# What ipv4_compat buys, and the only check that catches its loss: the router's
+# health check (dataplane.go) and the egress drainer (--envoy-admin-address)
+# dial 127.0.0.1, as literals in Go code that no manifest edit will follow.
+for app in atenet-router:9901 atenet-egress:15000; do
+  a="${app%:*}"; port="${app#*:}"
+  for loop in 127.0.0.1 ::1; do
+    # || true: a refused connection is a result to report, not a reason to die.
+    line="$(admin_get "${a}" "${loop}" "${port}" /ready | head -1 || true)"
+    case "${line}" in
+      "HTTP/1.1 200 OK") pass "${a}: ${loop}:${port}/ready -> 200" ;;
+      "") fail "${a}: ${loop}:${port}/ready -> no answer (socket not bound for this family)" ;;
+      *) fail "${a}: ${loop}:${port}/ready -> ${line}" ;;
+    esac
+  done
+done
+
+echo "== ipv4_compat, read from /config_dump =="
+# Never from /listeners: it reports resolved bound addresses and never emits
+# the flag, so a check written against it passes on a manifest that lost it.
+for app in atenet-router:9901:1 atenet-egress:15000:2; do
+  a="$(echo "${app}" | cut -d: -f1)"
+  port="$(echo "${app}" | cut -d: -f2)"
+  want="$(echo "${app}" | cut -d: -f3)"
+  got="$(grep -c '"ipv4_compat": true' <<<"$(admin_get "${a}" 127.0.0.1 "${port}" /config_dump)" || true)"
+  # Router: the admin socket only. Egress: admin plus the :443 listener, which
+  # Envoy reports twice (bootstrap and static views), so the floor is 2.
+  if [ "${got}" -ge "${want}" ]; then
+    pass "${a}: ${got} ipv4_compat socket(s) in /config_dump (want >= ${want})"
+  else
+    fail "${a}: ${got} ipv4_compat socket(s) in /config_dump, want >= ${want}"
+  fi
+done
+
+echo "== ingress listeners keep an IPv4 primary and gain a :: address =="
+# The asymmetry is deliberate and easy to "clean up" into a regression: these
+# are two sockets, so the v4 one stays as-is and ipv4_compat must be false on
+# the v6 one. The admin sockets above are one socket, so they need the flag.
+rdump="$(admin_get atenet-router 127.0.0.1 9901 /config_dump)"
+for l in ingress_http_listener:8080 ingress_https_listener:8443; do
+  name="${l%:*}"; port="${l#*:}"
+  block="$(sed -n "/\"name\": \"${name}\"/,/last_updated/p" <<<"${rdump}")"
+  prim=0; addl=0
+  contains "$(grep -A2 '"address": "0.0.0.0"' <<<"${block}" || true)" "\"port_value\": ${port}" && prim=1
+  contains "$(sed -n '/additional_addresses/,/]/p' <<<"${block}")" '"address": "::"' && addl=1
+  if [ "${prim}" = 1 ] && [ "${addl}" = 1 ]; then
+    pass "${name}: 0.0.0.0:${port} primary + :: additional"
+  else
+    fail "${name}: primary-0.0.0.0=${prim} additional-::=${addl}"
+  fi
+done
+
+echo "== a pod can reach each gateway on every family the cluster has =="
+# Blocking delete: a --wait=false here races its own re-create and the run
+# fails with AlreadyExists on any second invocation.
+${K} -n "${NS}" delete pod "${PROBE_POD}" --ignore-not-found --timeout=60s >/dev/null 2>&1 || true
+${K} -n "${NS}" run "${PROBE_POD}" --image="${PROBE_IMAGE}" --restart=Never \
+  --command -- sleep 600 >/dev/null
+trap '${K} -n "${NS}" delete pod "${PROBE_POD}" --ignore-not-found --wait=false >/dev/null 2>&1 || true' EXIT
+${K} -n "${NS}" wait --for=condition=Ready "pod/${PROBE_POD}" --timeout=120s >/dev/null
+
+# One ClusterIP per family, in the same order as .spec.ipFamilies.
+for app in atenet-router:80 atenet-egress:443; do
+  a="${app%:*}"; port="${app#*:}"
+  ips="$(${K} -n "${NS}" get svc "${a}" -o jsonpath='{.spec.clusterIPs}' | tr -d '[]"' | tr ',' ' ')"
+  i=0
+  for ip in ${ips}; do
+    fam="$(echo "${families}" | cut -d' ' -f$((i + 1)))"; i=$((i + 1))
+    # Bracket v6 literals for the URL; a bare colon would read as a port.
+    case "${ip}" in *:*) target="[${ip}]";; *) target="${ip}";; esac
+    # Connect, don't request: the router 404s without a Host header and egress
+    # :443 speaks mTLS CONNECT, so neither answers 200 when it is working.
+    if ${K} -n "${NS}" exec "${PROBE_POD}" -- \
+      timeout 10 nc -z "${ip}" "${port}" >/dev/null 2>&1; then
+      pass "${a} ${fam} ${target}:${port} accepts connections"
+    else
+      fail "${a} ${fam} ${target}:${port} refused -- Service has the IP, nothing listens on it"
+    fi
+  done
+  [ "${i}" -eq "$(echo "${families}" | wc -w | tr -d ' ')" ] \
+    || fail "${a}: ${i} ClusterIP(s) for $(echo "${families}" | wc -w | tr -d ' ') famil(ies) -- ipFamilyPolicy not PreferDualStack?"
+done
+
+echo
+if [ "${fails}" -eq 0 ]; then
+  echo "== PASS: both gateways bind and serve on ${families} =="
+else
+  echo "== FAIL: ${fails} check(s) failed =="
+  exit 1
+fi

hack/create-kind-cluster.sh

@@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}"
 KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}"
 reg_name="kind-registry"
 reg_port="${KIND_REGISTRY_PORT:-5001}"
+IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}"
 
 if [[ $# -gt 0 ]]; then
   case "$1" in
@@ -31,6 +32,8 @@ if [[ $# -gt 0 ]]; then
       echo "Configured through the environment:"
       echo "  KIND_CLUSTER_NAME  Name of the cluster to create (default: kind)."
       echo "  IP_FAMILY          Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)."
+      echo "  IPV6_DNS_UPSTREAM  Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6"
+      echo "                     (default: Google Public DNS). Override where those are unreachable."
       exit 0
       ;;
   esac
@@ -196,6 +199,55 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}
   docker network connect "kind" "${reg_name}"
 fi
 
+# 4.5. Give CoreDNS an IPv6 forwarder and a registry entry
+#
+# CoreDNS runs dnsPolicy: Default, inheriting the node's IPv4 resolver, which
+# no pod here can reach, so external lookups SERVFAIL. Step 3's registry
+# wiring is node-side, so it misses atelet too: that pull runs in atelet's own
+# netns, where "kind-registry" NXDOMAINs.
+if [[ "${IP_FAMILY}" == "ipv6" ]]; then
+  echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..."
+  reg_v6="$(docker inspect "${reg_name}" \
+    --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')"
+  if [[ -z "${reg_v6}" ]]; then
+    echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2
+    exit 1
+  fi
+
+  corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \
+    -o jsonpath='{.data.Corefile}')"
+  # fallthrough is load-bearing: without it every name but the registry
+  # NXDOMAINs, cluster.local included -- one outage traded for a worse one.
+  search="forward . /etc/resolv.conf"
+  replace="hosts {
+       ${reg_v6} ${reg_name}
+       fallthrough
+    }
+    forward . ${IPV6_DNS_UPSTREAM}"
+  # $search unquoted: bash 3.2 splices the quotes in literally. The block's
+  # first line inherits the matched line's indentation.
+  patched="${corefile/$search/$replace}"
+  if [[ "${patched}" == "${corefile}" ]]; then
+    echo "error: '${search}' not found in the CoreDNS Corefile" >&2
+    echo "       the Corefile layout changed upstream; update this block" >&2
+    exit 1
+  fi
+
+  # A YAML patch file avoids escaping the Corefile's newlines into JSON.
+  { printf 'data:\n  Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/    /'; } \
+    > "${ROOT}/bin/coredns-patch.yaml"
+  kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \
+    --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml"
+  kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns
+  kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \
+    --timeout=120s
+
+  # Its own script so it can be re-run against a live cluster: the hosts entry
+  # above is a snapshot of an address the registry can move off (#1049).
+  KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" REG_NAME="${reg_name}" \
+    IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM}" "${ROOT}"/hack/verify-ipv6-dns.sh
+fi
+
 # 5. Document the local registry in kube-public ConfigMap
 echo "Documenting local registry in cluster..."
 cat <<EOF | kubectl --context="${KUBECTL_CONTEXT}" apply -f -