From 7e33d4cb584d789ffa0c531581a3e8230b17d059 Mon Sep 17 00:00:00 2001 From: Scot Schuchert-Wells Date: Fri, 6 Nov 2026 19:01:44 +0000 Subject: [PATCH] fix(cni): Treat a shard reached only by default route as down When an egress shard's node dies, its SID route is withdrawn, but a compute node whose only other route is an upstream default over an SRv6 uplink still resolves the SID through that default. The shard keeps counting as alive, tenants stay pinned to it, their egress is lost for the whole outage, and a NAT64 standby is never promoted. Look up the FIB entry the SID matches and reject a default route, so a withdrawn SID makes its shard unreachable on every node. Kernels that cannot answer a FIB-match lookup keep the previous behaviour. --- docs/nat/active-active.md | 5 +- .../ebpf/egressroutemap/egressroute.go | 31 +++++++++++ .../egressroutemap/resolve_default_test.go | 53 +++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 internal/plumbing/ebpf/egressroutemap/resolve_default_test.go diff --git a/docs/nat/active-active.md b/docs/nat/active-active.md index c7c8f436..2e416a92 100644 --- a/docs/nat/active-active.md +++ b/docs/nat/active-active.md @@ -105,7 +105,10 @@ hashed mode with more than three NAT64 prefixes in - **A shard counts only while it is reachable.** A dead node's `EgressShard` status goes stale, but its BGP session drops and the route to its SID is withdrawn. A candidate takes tenants only while its SID resolves to a - neighbor on an SRv6 uplink. `galactic-cni` rebuilds the groups on every + neighbor on an SRv6 uplink through a route more specific than the + default: once the SID's route is withdrawn, a lookup that falls through + to a default route, even one over an uplink, counts as unreachable. + `galactic-cni` rebuilds the groups on every route or neighbor change that can affect a candidate, and every 30 seconds as a backstop, so failover takes as long as BGP takes to withdraw the route, plus a fraction of a second. diff --git a/internal/plumbing/ebpf/egressroutemap/egressroute.go b/internal/plumbing/ebpf/egressroutemap/egressroute.go index 6e5f33b9..dd2a0e15 100644 --- a/internal/plumbing/ebpf/egressroutemap/egressroute.go +++ b/internal/plumbing/ebpf/egressroutemap/egressroute.go @@ -183,6 +183,9 @@ func resolveLinkAndL2(sid net.IP) (linkIndex int, dmac, smac net.HardwareAddr, e "(the fabric has most likely not advertised this SID yet and the lookup fell through to a "+ "default route)", sid, linkIndex) } + if err := rejectDefaultRoute(sid); err != nil { + return 0, nil, nil, err + } nextHop := routes[0].Gw if nextHop == nil { nextHop = sid // gateway itself is on-link @@ -201,6 +204,34 @@ func resolveLinkAndL2(sid net.IP) (linkIndex int, dmac, smac net.HardwareAddr, e return linkIndex, dmac, smac, nil } +// rejectDefaultRoute fails when the route sid matches is a default route, +// even one leaving through an SRv6 uplink. The fabric advertises every SID it +// can reach as a specific route, so a lookup that only finds the default has +// lost that advertisement: the SID's node is gone, and the upstream router +// behind the default has never heard of its locator. Accepting it would keep +// a dead shard reachable and its tenants pinned to it for as long as the node +// is down. +// +// A kernel that cannot answer a FIB-match lookup cannot tell the two apart, so +// the check is skipped there rather than failing every SID. +func rejectDefaultRoute(sid net.IP) error { + matched, err := netlink.RouteGetWithOptions(sid, &netlink.RouteGetOptions{FIBMatch: true}) + if err != nil || len(matched) == 0 { + return nil + } + if dst := matched[0].Dst; dst == nil || isDefault(dst) { + return fmt.Errorf( + "no route to %s over an SRv6 uplink: only a default route matches it, so the fabric no longer "+ + "advertises it", sid) + } + return nil +} + +func isDefault(dst *net.IPNet) bool { + ones, _ := dst.Mask.Size() + return ones == 0 +} + // resolveNeighbor returns nextHop's resolved L2 address on linkIndex, actively // soliciting the kernel to populate it when a passive cache read finds // nothing. diff --git a/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go b/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go new file mode 100644 index 00000000..800f2b3e --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "fmt" + "net" + "strings" + "testing" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" +) + +func TestResolveLinkAndL2_RejectsADefaultRouteOverAnUplink(t *testing.T) { + requireRoot(t) + + nearNS, linkIndex, farIP := setUpVethPair(t, + "vrfdr0", "fd00:beef:3::1/64", + "vrfdr0p", "fd00:beef:3::2/64") + + restore := uplinkIndexesFn + uplinkIndexesFn = func() (map[int]struct{}, error) { return map[int]struct{}{linkIndex: {}}, nil } + t.Cleanup(func() { uplinkIndexesFn = restore }) + + sid := net.ParseIP("2001:db8:ff02:2002:e001::") + _, locator, _ := net.ParseCIDR("2001:db8:ff02:2002::/64") + + var defaultErr, specificErr error + err := nearNS.Do(func(_ ns.NetNS) error { + if err := netlink.RouteAdd(&netlink.Route{LinkIndex: linkIndex, Gw: farIP}); err != nil { + return fmt.Errorf("add default route: %w", err) + } + _, _, _, defaultErr = resolveLinkAndL2(sid) + + if err := netlink.RouteAdd(&netlink.Route{LinkIndex: linkIndex, Dst: locator, Gw: farIP}); err != nil { + return fmt.Errorf("add locator route: %w", err) + } + _, _, _, specificErr = resolveLinkAndL2(sid) + return nil + }) + if err != nil { + t.Fatal(err) + } + if defaultErr == nil || !strings.Contains(defaultErr.Error(), "only a default route matches") { + t.Errorf("resolve through only a default route over an uplink: err = %v, want a default-route rejection", defaultErr) + } + if specificErr != nil { + t.Errorf("resolve through the advertised locator: %v", specificErr) + } +}