From 6b93e41cc861e5bf9175b295a368ffa668db0b31 Mon Sep 17 00:00:00 2001 From: Scot Schuchert-Wells Date: Tue, 10 Nov 2026 00:29:13 +0000 Subject: [PATCH] fix(vrf): Accept a default route to backend SIDs in the sidecar The ingress sidecar resolves each backend's SID from Envoy's pod network namespace, whose only route is the default toward the host. The check that treats a SID reached only by a default route as down, added for egress shard failover, therefore refuses every backend there: no egress route is installed and Envoy cannot reach any VPC backend. Let a process opt out of that check, and have the sidecar's kernel backend do so. Every other caller keeps rejecting a default-route match, so shard failover is unchanged. --- internal/ingresssidecar/backend.go | 11 ++++- internal/ingresssidecar/backend_test.go | 20 ++++++++ .../ebpf/egressroutemap/egressroute.go | 27 +++++++++- .../egressroutemap/resolve_default_test.go | 49 +++++++++++++++++++ 4 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 internal/ingresssidecar/backend_test.go diff --git a/internal/ingresssidecar/backend.go b/internal/ingresssidecar/backend.go index 9a6f362f..39cae467 100644 --- a/internal/ingresssidecar/backend.go +++ b/internal/ingresssidecar/backend.go @@ -11,6 +11,7 @@ import ( "regexp" "strings" + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" "go.datum.net/galactic/internal/plumbing/srv6" "go.datum.net/galactic/internal/plumbing/vrf" ) @@ -80,7 +81,15 @@ type kernelBackend struct{} // NewKernelBackend returns the production Backend, wired to real kernel state. // Requires CAP_NET_ADMIN. -func NewKernelBackend() Backend { return kernelBackend{} } +// +// It also makes the process accept a default route to a backend's SID. The +// sidecar resolves SIDs in Envoy's pod namespace, where the default toward the +// host is the only route, so the shard check that rejects a default-route +// match would refuse every backend. +func NewKernelBackend() Backend { + egressroutemap.AcceptDefaultRoute() + return kernelBackend{} +} func (kernelBackend) EnsureVRF(vpc string) (uint32, error) { if err := vrf.AddInRange(vpc, sidecarTableIDMin, sidecarTableIDMax); err != nil { diff --git a/internal/ingresssidecar/backend_test.go b/internal/ingresssidecar/backend_test.go new file mode 100644 index 00000000..dc70130c --- /dev/null +++ b/internal/ingresssidecar/backend_test.go @@ -0,0 +1,20 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingresssidecar + +import ( + "testing" + + "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" +) + +func TestNewKernelBackend_AcceptsADefaultRouteToBackendSIDs(t *testing.T) { + NewKernelBackend() + + if !egressroutemap.DefaultRouteAccepted() { + t.Error("the kernel backend left default routes rejected, so every backend SID resolved from " + + "Envoy's pod namespace, whose only route is the default, would be refused") + } +} diff --git a/internal/plumbing/ebpf/egressroutemap/egressroute.go b/internal/plumbing/ebpf/egressroutemap/egressroute.go index dd2a0e15..061e6a2a 100644 --- a/internal/plumbing/ebpf/egressroutemap/egressroute.go +++ b/internal/plumbing/ebpf/egressroutemap/egressroute.go @@ -13,6 +13,7 @@ import ( "net" "net/netip" "strconv" + "sync/atomic" "time" "github.com/cilium/ebpf" @@ -213,8 +214,12 @@ func resolveLinkAndL2(sid net.IP) (linkIndex int, dmac, smac net.HardwareAddr, e // is down. // // A kernel that cannot answer a FIB-match lookup cannot tell the two apart, so -// the check is skipped there rather than failing every SID. +// the check is skipped there rather than failing every SID. So is a process +// that called AcceptDefaultRoute. func rejectDefaultRoute(sid net.IP) error { + if defaultRouteAccepted.Load() { + return nil + } matched, err := netlink.RouteGetWithOptions(sid, &netlink.RouteGetOptions{FIBMatch: true}) if err != nil || len(matched) == 0 { return nil @@ -227,6 +232,26 @@ func rejectDefaultRoute(sid net.IP) error { return nil } +// defaultRouteAccepted is set by AcceptDefaultRoute. +var defaultRouteAccepted atomic.Bool + +// AcceptDefaultRoute makes a default route an acceptable route to any SID for +// the rest of the process, turning off rejectDefaultRoute. +// +// It is for a process that resolves SIDs from a network namespace that never +// holds fabric routes. The ingress sidecar runs in Envoy's pod namespace, whose +// only route is the default toward the host, and the host carries the fabric's +// specific routes. There a default-route match is the only answer a reachable +// SID can ever give, so rejecting it would refuse every backend. +func AcceptDefaultRoute() { + defaultRouteAccepted.Store(true) +} + +// DefaultRouteAccepted reports whether AcceptDefaultRoute has been called. +func DefaultRouteAccepted() bool { + return defaultRouteAccepted.Load() +} + func isDefault(dst *net.IPNet) bool { ones, _ := dst.Mask.Size() return ones == 0 diff --git a/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go b/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go index 800f2b3e..0d954a89 100644 --- a/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go +++ b/internal/plumbing/ebpf/egressroutemap/resolve_default_test.go @@ -51,3 +51,52 @@ func TestResolveLinkAndL2_RejectsADefaultRouteOverAnUplink(t *testing.T) { t.Errorf("resolve through the advertised locator: %v", specificErr) } } + +func TestResolveLinkAndL2_AcceptsADefaultRouteWhenTheProcessAllowsIt(t *testing.T) { + requireRoot(t) + + nearNS, linkIndex, farIP := setUpVethPair(t, + "vrfdr1", "fd00:beef:4::1/64", + "vrfdr1p", "fd00:beef:4::2/64") + + restore := uplinkIndexesFn + uplinkIndexesFn = func() (map[int]struct{}, error) { return map[int]struct{}{linkIndex: {}}, nil } + t.Cleanup(func() { uplinkIndexesFn = restore }) + + acceptDefaultRouteForTest(t) + + sid := net.ParseIP("2001:db8:ff02:2002:e001::") + + var resolveErr error + err := nearNS.Do(func(_ ns.NetNS) error { + if err := netlink.RouteAdd(&netlink.Route{LinkIndex: linkIndex, Gw: farIP}); err != nil { + return fmt.Errorf("add default route: %w", err) + } + _, _, _, resolveErr = resolveLinkAndL2(sid) + return nil + }) + if err != nil { + t.Fatal(err) + } + if resolveErr != nil { + t.Errorf("resolve through only a default route with default routes accepted: %v", resolveErr) + } +} + +func TestRejectDefaultRoute_SkippedOnceDefaultRoutesAreAccepted(t *testing.T) { + acceptDefaultRouteForTest(t) + + if !DefaultRouteAccepted() { + t.Fatal("DefaultRouteAccepted() = false after AcceptDefaultRoute") + } + if err := rejectDefaultRoute(net.ParseIP("2001:db8:ff02:2002:e001::")); err != nil { + t.Errorf("rejectDefaultRoute with default routes accepted: %v", err) + } +} + +func acceptDefaultRouteForTest(t *testing.T) { + t.Helper() + prev := defaultRouteAccepted.Load() + AcceptDefaultRoute() + t.Cleanup(func() { defaultRouteAccepted.Store(prev) }) +}