diff --git a/docs/nat/active-active.md b/docs/nat/active-active.md index c7c8f436..2e416a92 100644 --- a/docs/nat/active-active.md +++ b/docs/nat/active-active.md @@ -105,7 +105,10 @@ hashed mode with more than three NAT64 prefixes in - **A shard counts only while it is reachable.** A dead node's `EgressShard` status goes stale, but its BGP session drops and the route to its SID is withdrawn. A candidate takes tenants only while its SID resolves to a - neighbor on an SRv6 uplink. `galactic-cni` rebuilds the groups on every + neighbor on an SRv6 uplink through a route more specific than the + default: once the SID's route is withdrawn, a lookup that falls through + to a default route, even one over an uplink, counts as unreachable. + `galactic-cni` rebuilds the groups on every route or neighbor change that can affect a candidate, and every 30 seconds as a backstop, so failover takes as long as BGP takes to withdraw the route, plus a fraction of a second. diff --git a/internal/plumbing/ebpf/egressroutemap/egressroute.go b/internal/plumbing/ebpf/egressroutemap/egressroute.go index 6e5f33b9..dd2a0e15 100644 --- a/internal/plumbing/ebpf/egressroutemap/egressroute.go +++ b/internal/plumbing/ebpf/egressroutemap/egressroute.go @@ -183,6 +183,9 @@ func resolveLinkAndL2(sid net.IP) (linkIndex int, dmac, smac net.HardwareAddr, e "(the fabric has most likely not advertised this SID yet and the lookup fell through to a "+ "default route)", sid, linkIndex) } + if err := rejectDefaultRoute(sid); err != nil { + return 0, nil, nil, err + } nextHop := routes[0].Gw if nextHop == nil { nextHop = sid // gateway itself is on-link @@ -201,6 +204,34 @@ func resolveLinkAndL2(sid net.IP) (linkIndex int, dmac, smac net.HardwareAddr, e return linkIndex, dmac, smac, nil } +// rejectDefaultRoute fails when the route sid matches is a default route, +// even one leaving through an SRv6 uplink. The fabric advertises every SID it +// can reach as a specific route, so a lookup that only finds the default has +// lost that advertisement: the SID's node is gone, and the upstream router +// behind the default has never heard of its locator. Accepting it would keep +// a dead shard reachable and its tenants pinned to it for as long as the node +// is down. +// +// A kernel that cannot answer a FIB-match lookup cannot tell the two apart, so +// the check is skipped there rather than failing every SID. +func rejectDefaultRoute(sid net.IP) error { + matched, err := netlink.RouteGetWithOptions(sid, &netlink.RouteGetOptions{FIBMatch: true}) + if err != nil || len(matched) == 0 { + return nil + } + if dst := matched[0].Dst; dst == nil || isDefault(dst) { + return fmt.Errorf( + "no route to %s over an SRv6 uplink: only a default route matches it, so the fabric no longer "+ + "advertises it", sid) + } + return nil +} + +func isDefault(dst *net.IPNet) bool { + ones, _ := dst.Mask.Size() + return ones == 0 +} + // resolveNeighbor returns nextHop's resolved L2 address on linkIndex, actively // soliciting the kernel to populate it when a passive cache read finds // nothing. diff --git a/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go b/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go new file mode 100644 index 00000000..800f2b3e --- /dev/null +++ b/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package egressroutemap + +import ( + "fmt" + "net" + "strings" + "testing" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" +) + +func TestResolveLinkAndL2_RejectsADefaultRouteOverAnUplink(t *testing.T) { + requireRoot(t) + + nearNS, linkIndex, farIP := setUpVethPair(t, + "vrfdr0", "fd00:beef:3::1/64", + "vrfdr0p", "fd00:beef:3::2/64") + + restore := uplinkIndexesFn + uplinkIndexesFn = func() (map[int]struct{}, error) { return map[int]struct{}{linkIndex: {}}, nil } + t.Cleanup(func() { uplinkIndexesFn = restore }) + + sid := net.ParseIP("2001:db8:ff02:2002:e001::") + _, locator, _ := net.ParseCIDR("2001:db8:ff02:2002::/64") + + var defaultErr, specificErr error + err := nearNS.Do(func(_ ns.NetNS) error { + if err := netlink.RouteAdd(&netlink.Route{LinkIndex: linkIndex, Gw: farIP}); err != nil { + return fmt.Errorf("add default route: %w", err) + } + _, _, _, defaultErr = resolveLinkAndL2(sid) + + if err := netlink.RouteAdd(&netlink.Route{LinkIndex: linkIndex, Dst: locator, Gw: farIP}); err != nil { + return fmt.Errorf("add locator route: %w", err) + } + _, _, _, specificErr = resolveLinkAndL2(sid) + return nil + }) + if err != nil { + t.Fatal(err) + } + if defaultErr == nil || !strings.Contains(defaultErr.Error(), "only a default route matches") { + t.Errorf("resolve through only a default route over an uplink: err = %v, want a default-route rejection", defaultErr) + } + if specificErr != nil { + t.Errorf("resolve through the advertised locator: %v", specificErr) + } +}