diff --git a/lib/ipip.go b/lib/ipip.go new file mode 100644 index 0000000..dcf069b --- /dev/null +++ b/lib/ipip.go @@ -0,0 +1,620 @@ +package lib + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "net" + "net/http" + "net/netip" + "strconv" + "strings" + "time" + + "github.com/vishvananda/netlink" +) + +// ipipPeer tracks server-side state for a single IPIP tunnel peer. +// +// One ipipPeer corresponds to exactly one Linux IPIP interface created on the +// server, with the peer's HTTPS source address as the remote tunnel endpoint +// and an inner IP allocated from srv.WgCidr. +type ipipPeer struct { + clientIP netip.Addr // outer (HTTPS) source address of the client + peerIP netip.Addr // inner address allocated from srv.WgCidr + ifname string // Linux interface name (e.g. vp0-1) + + lastSeen time.Time // last time we observed activity from this peer + rxBytes uint64 // rx bytes observed at lastSeen, for activity detection + txBytes uint64 // tx bytes observed at lastSeen, for activity detection +} + +type connectIpipResponse struct { + AssignedAddr string +} + +// ipipIfnameMaxLen is the maximum visible length of a Linux interface name +// (IFNAMSIZ is 16 including the null terminator). +const ipipIfnameMaxLen = 15 + +// ipipIfname returns the Linux interface name used for the IPIP tunnel to +// the peer at peerIP. +// +// The name is "vp-" where offset is the peer's distance +// from srv.WgCidr.Addr(). Using the full offset from the CIDR base (rather +// than e.g. the low 16 bits of peerIP) keeps the suffix globally unique +// across any allowed WgCidr width: for two distinct peers in the same +// server's CIDR, their offsets necessarily differ. The "vp-" prefix +// also gives us a stable wildcard (vp-+) for iptables. +// +// The function returns an error if the resulting name would exceed +// IFNAMSIZ. This never fires for production-sized CIDRs -- a /16 yields at +// most a 5-digit offset, well within budget alongside a 5-digit server +// index -- but it is a hard runtime guard against a CIDR wide enough +// (roughly /8 or wider, paired with a large server index) that the offset +// pushes the name past IFNAMSIZ, where kernel truncation could make two +// distinct peers collide on the same interface name. +func (srv *Server) ipipIfname(peerIP netip.Addr) (string, error) { + base := srv.WgCidr.Addr().As4() + p := peerIP.As4() + baseInt := uint32(base[0])<<24 | uint32(base[1])<<16 | + uint32(base[2])<<8 | uint32(base[3]) + peerInt := uint32(p[0])<<24 | uint32(p[1])<<16 | + uint32(p[2])<<8 | uint32(p[3]) + offset := peerInt - baseInt + name := fmt.Sprintf("vp%d-%d", srv.Index, offset) + if len(name) > ipipIfnameMaxLen { + return "", fmt.Errorf( + "ipip ifname %q exceeds IFNAMSIZ (%d > %d); WgCidr is too wide for IPIP", + name, len(name), ipipIfnameMaxLen) + } + return name, nil +} + +// ipipIfaceWildcard returns the iptables-style wildcard that matches every +// IPIP interface created for this server. +func (srv *Server) ipipIfaceWildcard() string { + return fmt.Sprintf("vp%d-+", srv.Index) +} + +// ipipPeerFromIfname is the inverse of ipipIfname: it recovers the peer's +// inner IP from a "vp-" interface name. The startup sweep +// uses it to remove the per-peer iptables rules of tunnels left over from a +// previous process. Returns false if the name does not belong to this +// server's IPIP interfaces. +func (srv *Server) ipipPeerFromIfname(ifname string) (netip.Addr, bool) { + suffix, found := strings.CutPrefix(ifname, fmt.Sprintf("vp%d-", srv.Index)) + if !found { + return netip.Addr{}, false + } + offset, err := strconv.ParseUint(suffix, 10, 32) + if err != nil { + return netip.Addr{}, false + } + base := srv.WgCidr.Addr().As4() + baseInt := uint32(base[0])<<24 | uint32(base[1])<<16 | + uint32(base[2])<<8 | uint32(base[3]) + peerInt := baseInt + uint32(offset) + return netip.AddrFrom4([4]byte{ + byte(peerInt >> 24), byte(peerInt >> 16), + byte(peerInt >> 8), byte(peerInt), + }), true +} + +// SweepStaleIpip removes IPIP interfaces (and their per-peer iptables rules) +// left over from a previous process. CleanupIpip only runs on a clean +// shutdown; after a crash or SIGKILL the interfaces survive while the new +// process starts with an empty allocator, so a leftover /32 host route could +// blackhole an IP the allocator later hands to a new WireGuard or IPIP peer. +// There is no adopt-on-restart path for IPIP (unlike WireGuard), so any +// surviving vp-* tunnel is stale by definition. +func (srv *Server) SweepStaleIpip() error { + links, err := netlink.LinkList() + if err != nil { + return fmt.Errorf("list links for ipip sweep: %v", err) + } + for _, link := range links { + ifname := link.Attrs().Name + peerIP, ok := srv.ipipPeerFromIfname(ifname) + if !ok { + continue + } + if _, isIptun := link.(*netlink.Iptun); !isIptun { + continue + } + log.Printf("[%v] sweeping stale ipip tunnel %s (peer %v)", + srv.BindAddr, ifname, peerIP) + srv.tearDownIpipLink(ifname, peerIP) + } + return nil +} + +// connectIpipHandler handles POST /connect-ipip. +// +// It authenticates the request with the shared Bearer password (matching the +// existing /connect handler), allocates an inner IP from srv.ipAllocator, +// creates a Linux IPIP tunnel whose remote is the HTTPS source address of the +// request, installs a host route so that return traffic destined for the +// inner IP exits via that tunnel, and returns the assigned inner address. +// +// Repeated calls from the same client IP are idempotent: the existing peer +// is reused and its lastSeen timestamp refreshed. +func (srv *Server) connectIpipHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if r.Header.Get("Authorization") != "Bearer "+srv.Password { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + http.Error(w, "invalid remote address", http.StatusBadRequest) + return + } + clientIP, err := netip.ParseAddr(host) + if err != nil { + http.Error(w, "invalid remote address", http.StatusBadRequest) + return + } + if !clientIP.Is4() { + http.Error(w, "ipv4 client address required", http.StatusBadRequest) + return + } + + // Idempotent path: if we already have an IPIP tunnel for this client + // IP and its kernel interface still exists, refresh lastSeen and + // return the same assignment. If the interface vanished out-of-band + // (manual `ip link del`, kernel reload, etc.) we cannot just refresh + // lastSeen: a retrying client would keep the stale entry alive + // forever, since removeIdleIpipPeers's freshness guard skips a peer + // whose lastSeen was just bumped. Drop the stale entry and fall + // through to fresh allocation. + srv.ipipMu.Lock() + existing, ok := srv.ipipPeers[clientIP] + srv.ipipMu.Unlock() + + if ok { + // Probe the interface without holding ipipMu, so the netlink + // syscall doesn't block the idle loop or other requests. Only a + // genuine "not found" counts as vanished: a transient lookup + // failure must not tear down a working tunnel. + _, lookupErr := netlink.LinkByName(existing.ifname) + var notFound netlink.LinkNotFoundError + vanished := errors.As(lookupErr, ¬Found) + if lookupErr != nil && !vanished { + log.Printf("[%v] ipip iface %s lookup failed transiently (%v); reusing", + srv.BindAddr, existing.ifname, lookupErr) + } + + // Re-check identity under the lock: the entry may have been + // reaped or replaced while ipipMu was released for the probe. + srv.ipipMu.Lock() + if current, stillOurs := srv.ipipPeers[clientIP]; !stillOurs || current != existing { + // Entry changed under us; fall through to fresh allocation + // (the race re-check below reconciles with the winner). + srv.ipipMu.Unlock() + } else if !vanished { + current.lastSeen = time.Now() + srv.ipipMu.Unlock() + writeIpipResponse(w, fmt.Sprintf("%v/%d", existing.peerIP, srv.WgCidr.Bits())) + return + } else { + delete(srv.ipipPeers, clientIP) + srv.ipipMu.Unlock() + log.Printf("[%v] ipip iface %s for %v vanished; rebuilding", + srv.BindAddr, existing.ifname, clientIP) + srv.removeIpipPeerFilter(existing.ifname, existing.peerIP) + srv.ipAllocator.Free(existing.peerIP) + } + } + + assigned, errMsg, errStatus := srv.createIpipPeer(clientIP) + if errMsg != "" { + http.Error(w, errMsg, errStatus) + return + } + writeIpipResponse(w, assigned) +} + +// createIpipPeer allocates an inner IP for clientIP, creates the IPIP +// tunnel, and registers the peer. On success it returns the assigned inner +// address in CIDR notation; on failure it returns a non-empty errMsg and +// the HTTP status to report. Writing the response is left to the caller so +// ipipCreateMu is released before any client I/O: a slow client read must +// not stall every other /connect-ipip create. +// +// Tunnel creation is serialized by ipipCreateMu. The kernel permits only +// one IPIP tunnel per (local, remote) pair, so two parallel first-time +// requests from the same client IP would race: both miss the peer map, both +// allocate, and the loser's LinkAdd fails with EEXIST even though a working +// tunnel exists. Holding the create lock across allocate+create+insert lets +// the second request observe the winner's entry below instead of colliding +// in the kernel. +func (srv *Server) createIpipPeer(clientIP netip.Addr) (assigned, errMsg string, errStatus int) { + srv.ipipCreateMu.Lock() + defer srv.ipipCreateMu.Unlock() + + srv.ipipMu.Lock() + if winner, ok := srv.ipipPeers[clientIP]; ok { + winner.lastSeen = time.Now() + srv.ipipMu.Unlock() + return fmt.Sprintf("%v/%d", winner.peerIP, srv.WgCidr.Bits()), "", 0 + } + srv.ipipMu.Unlock() + + peerIP := srv.ipAllocator.Allocate() + if peerIP.IsUnspecified() { + log.Printf("no more ip addresses available in %v", srv.WgCidr) + return "", "no more IP addresses available", http.StatusServiceUnavailable + } + + ifname, err := srv.ipipIfname(peerIP) + if err != nil { + srv.ipAllocator.Free(peerIP) + log.Printf("[%v] %v", srv.BindAddr, err) + return "", "ipip ifname out of range", http.StatusInternalServerError + } + if err := srv.createIpipLink(ifname, clientIP, peerIP); err != nil { + srv.ipAllocator.Free(peerIP) + log.Printf("[%v] failed to create IPIP tunnel for %v: %v", + srv.BindAddr, clientIP, err) + return "", "failed to create IPIP tunnel", http.StatusInternalServerError + } + + peer := &ipipPeer{ + clientIP: clientIP, + peerIP: peerIP, + ifname: ifname, + lastSeen: time.Now(), + } + + // Belt-and-braces: creation is serialized by ipipCreateMu, so no other + // request should have inserted an entry for this client IP since the + // re-check above, but if one somehow did, drop the tunnel we just + // built and reuse the winner so we don't leak an allocation or an + // interface. Likewise, if CleanupIpip already ran (this handler + // outlived the shutdown drain), registering the peer now would leak a + // tunnel that nothing will ever tear down. + srv.ipipMu.Lock() + if srv.ipipClosed { + srv.ipipMu.Unlock() + srv.tearDownIpipLink(ifname, peerIP) + srv.ipAllocator.Free(peerIP) + return "", "server shutting down", http.StatusServiceUnavailable + } + if winner, ok := srv.ipipPeers[clientIP]; ok { + winner.lastSeen = time.Now() + srv.ipipMu.Unlock() + srv.tearDownIpipLink(ifname, peerIP) + srv.ipAllocator.Free(peerIP) + return fmt.Sprintf("%v/%d", winner.peerIP, srv.WgCidr.Bits()), "", 0 + } + srv.ipipPeers[clientIP] = peer + srv.ipipMu.Unlock() + + log.Printf("[%v] new ipip peer %v at %v (iface %s)", + srv.BindAddr, clientIP, peerIP, ifname) + + return fmt.Sprintf("%v/%d", peerIP, srv.WgCidr.Bits()), "", 0 +} + +func writeIpipResponse(w http.ResponseWriter, assigned string) { + resp := &connectIpipResponse{AssignedAddr: assigned} + respBuf, err := json.Marshal(resp) + if err != nil { + http.Error(w, "failed to serialize response", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(respBuf) +} + +// createIpipLink creates the Linux IPIP interface and installs a host route +// pointing peerIP at it so return traffic finds the right tunnel. +// +// We intentionally do NOT add an IP address to the IPIP interface: the +// equivalent assignment in the WireGuard path puts srv.WgCidr.Addr() on the +// WireGuard interface, and adding the same local address to a second +// interface would either be rejected or break routing. A more-specific /32 +// route per peer is enough to deliver decapsulated return traffic to the +// correct tunnel, and the MASQUERADE rule already in place on srv.BindIface +// handles outbound NAT. +func (srv *Server) createIpipLink(ifname string, remote, peerIP netip.Addr) error { + link := &netlink.Iptun{ + LinkAttrs: netlink.LinkAttrs{Name: ifname}, + Local: addrToIp(srv.BindAddr), + Remote: addrToIp(remote), + } + // Best-effort cleanup of any stale interface with the same name. + _ = netlink.LinkDel(link) + + if err := netlink.LinkAdd(link); err != nil { + return fmt.Errorf("add ipip link: %v", err) + } + + // netlink.LinkAdd does not populate the struct's Index field, so look up + // the link by name to get the kernel-assigned attributes (including the + // real ifindex) before we install routes that depend on it. + resolved, err := netlink.LinkByName(ifname) + if err != nil { + _ = netlink.LinkDel(link) + return fmt.Errorf("resolve ipip link %s: %v", ifname, err) + } + + if err := netlink.LinkSetUp(resolved); err != nil { + _ = netlink.LinkDel(resolved) + return fmt.Errorf("bring up ipip link: %v", err) + } + + // Pin the peer's inner IP to this tunnel so decapsulated return packets + // go back out the right interface (a /32 wins over the /16 connected + // route on the WireGuard interface). + dst := prefixToIPNet(netip.PrefixFrom(peerIP, 32)) + route := &netlink.Route{ + LinkIndex: resolved.Attrs().Index, + Dst: &dst, + Scope: netlink.SCOPE_LINK, + } + if err := netlink.RouteReplace(route); err != nil { + _ = netlink.LinkDel(resolved) + return fmt.Errorf("add host route for %v: %v", peerIP, err) + } + + // Per-peer iptables filter: only accept forwarded traffic whose inner + // source IP matches the peer we assigned this tunnel to, and drop + // everything else arriving on this interface. A decapsulated IPIP + // packet carries an attacker-controlled inner source, so this is what + // stops a peer from injecting transit traffic claiming to be from a + // different peer's inner IP. Inner packets that terminate on the host + // itself go through INPUT instead, where ufw's default deny and + // connectIpipHandler's check that the request did not originate from + // inside srv.WgCidr together cover the control-plane concern. + if err := srv.addIpipPeerFilter(ifname, peerIP); err != nil { + _ = netlink.LinkDel(resolved) + return fmt.Errorf("install ipip peer filter: %v", err) + } + + return nil +} + +// tearDownIpipLink removes the per-peer iptables filter and the IPIP +// interface. The interface deletion is what carries the kernel state; +// removing the iptables rules first keeps them from referencing a vanished +// interface for the brief window before they're cleaned up. +func (srv *Server) tearDownIpipLink(ifname string, peerIP netip.Addr) { + srv.removeIpipPeerFilter(ifname, peerIP) + + link, err := netlink.LinkByName(ifname) + if err != nil { + // Already gone; nothing to do. + return + } + if err := netlink.LinkDel(link); err != nil { + log.Printf("[%v] failed to delete ipip link %s: %v", + srv.BindAddr, ifname, err) + } +} + +// ipipPeerAcceptRule is the iptables rule that permits forwarded traffic +// arriving on this peer's IPIP interface with the expected inner source IP. +func ipipPeerAcceptRule(ifname string, peerIP netip.Addr) []string { + return []string{ + "-i", ifname, + "-s", fmt.Sprintf("%v/32", peerIP), + "-j", "ACCEPT", + "-m", "comment", "--comment", + fmt.Sprintf("vprox ipip accept peer %v on %s", peerIP, ifname), + } +} + +// ipipPeerDropRule is the iptables rule that drops anything else arriving +// on this peer's IPIP interface (i.e. inner source spoofing). +func ipipPeerDropRule(ifname string) []string { + return []string{ + "-i", ifname, + "-j", "DROP", + "-m", "comment", "--comment", + fmt.Sprintf("vprox ipip drop spoofed on %s", ifname), + } +} + +func (srv *Server) addIpipPeerFilter(ifname string, peerIP netip.Addr) error { + accept := ipipPeerAcceptRule(ifname, peerIP) + if err := srv.Ipt.AppendUnique("filter", "FORWARD", accept...); err != nil { + return fmt.Errorf("add ipip accept rule: %v", err) + } + drop := ipipPeerDropRule(ifname) + if err := srv.Ipt.AppendUnique("filter", "FORWARD", drop...); err != nil { + _ = srv.Ipt.Delete("filter", "FORWARD", accept...) + return fmt.Errorf("add ipip drop rule: %v", err) + } + return nil +} + +func (srv *Server) removeIpipPeerFilter(ifname string, peerIP netip.Addr) { + // The interface is about to be deleted (or already is), so removal + // order doesn't matter for security; either rule alone matches + // nothing once the interface is gone. Use DeleteIfExists so a + // partially-installed filter (e.g. failed mid-add) cleans up + // without a noisy "rule does not exist" error. + if err := srv.Ipt.DeleteIfExists("filter", "FORWARD", ipipPeerDropRule(ifname)...); err != nil { + log.Printf("[%v] failed to remove ipip drop rule for %s: %v", + srv.BindAddr, ifname, err) + } + if err := srv.Ipt.DeleteIfExists("filter", "FORWARD", ipipPeerAcceptRule(ifname, peerIP)...); err != nil { + log.Printf("[%v] failed to remove ipip accept rule for %s: %v", + srv.BindAddr, ifname, err) + } +} + +func (srv *Server) removeIdleIpipPeersLoop() { + for { + select { + case <-srv.Ctx.Done(): + return + case <-time.After(5 * time.Second): + } + + srv.removeIdleIpipPeers() + } +} + +// removeIdleIpipPeers prunes IPIP peers whose tunnel has seen no traffic for +// longer than PeerIdleTimeout. Activity is detected by polling the +// interface's rx_bytes and tx_bytes counters via netlink; counting both +// directions means a peer in the middle of a one-way transfer (e.g. a +// download with little return traffic) is not pruned mid-stream. +func (srv *Server) removeIdleIpipPeers() { + srv.ipipMu.Lock() + type snapshot struct { + clientIP netip.Addr + ifname string + peer *ipipPeer + } + snaps := make([]snapshot, 0, len(srv.ipipPeers)) + for clientIP, peer := range srv.ipipPeers { + snaps = append(snaps, snapshot{clientIP: clientIP, ifname: peer.ifname, peer: peer}) + } + srv.ipipMu.Unlock() + + now := time.Now() + type removal struct { + snapshot + vanished bool + } + var toRemove []removal + for _, s := range snaps { + link, err := netlink.LinkByName(s.ifname) + if err != nil { + // Only a genuine "not found" counts as vanished. A + // transient lookup failure (kernel resource pressure, brief + // netlink glitch, etc.) must NOT tear down an + // actively-used tunnel, since vanished entries bypass the + // lastSeen freshness guard below. Log and re-check next + // poll instead. Matches the discrimination in + // connectIpipHandler. + var notFound netlink.LinkNotFoundError + if errors.As(err, ¬Found) { + toRemove = append(toRemove, removal{snapshot: s, vanished: true}) + } else { + log.Printf("[%v] ipip iface %s lookup failed transiently (%v); will retry", + srv.BindAddr, s.ifname, err) + } + continue + } + stats := link.Attrs().Statistics + var rx, tx uint64 + if stats != nil { + rx = stats.RxBytes + tx = stats.TxBytes + } + + srv.ipipMu.Lock() + if rx != s.peer.rxBytes || tx != s.peer.txBytes { + s.peer.rxBytes = rx + s.peer.txBytes = tx + s.peer.lastSeen = now + } + idle := now.Sub(s.peer.lastSeen) > PeerIdleTimeout + srv.ipipMu.Unlock() + + if idle { + toRemove = append(toRemove, removal{snapshot: s}) + } + } + + for _, r := range toRemove { + srv.ipipMu.Lock() + // Re-check inside the lock in case a fresh /connect-ipip from the + // same client IP just replaced this entry. + current, ok := srv.ipipPeers[r.clientIP] + if !ok || current != r.peer { + srv.ipipMu.Unlock() + continue + } + // For idle-timeout removals, give a racing /connect-ipip that + // bumped lastSeen the benefit of the doubt. For vanished + // interfaces there's nothing to keep alive: the kernel state + // is gone, so reap regardless of lastSeen. + if !r.vanished && now.Sub(current.lastSeen) <= PeerIdleTimeout { + srv.ipipMu.Unlock() + continue + } + delete(srv.ipipPeers, r.clientIP) + srv.ipipMu.Unlock() + + reason := "idle" + if r.vanished { + reason = "vanished" + } + log.Printf("[%v] removing %s ipip peer %v at %v", + srv.BindAddr, reason, r.clientIP, r.peer.peerIP) + srv.tearDownIpipLink(r.peer.ifname, r.peer.peerIP) + srv.ipAllocator.Free(r.peer.peerIP) + } +} + +// CleanupIpip tears down every IPIP interface this server created. It is +// safe to call multiple times. +func (srv *Server) CleanupIpip() { + srv.ipipMu.Lock() + srv.ipipClosed = true + peers := make([]*ipipPeer, 0, len(srv.ipipPeers)) + for _, p := range srv.ipipPeers { + peers = append(peers, p) + } + srv.ipipPeers = make(map[netip.Addr]*ipipPeer) + srv.ipipMu.Unlock() + + for _, p := range peers { + srv.tearDownIpipLink(p.ifname, p.peerIP) + srv.ipAllocator.Free(p.peerIP) + } +} + +// iptablesIpipMssRules adds or removes TCP MSS clamping for the IPIP +// interfaces (inbound and outbound) so traffic fits the tunnel MTU. +func (srv *Server) iptablesIpipMssRules(enabled bool) error { + out := []string{ + "-o", srv.ipipIfaceWildcard(), + "-p", "tcp", + "--tcp-flags", "SYN,RST", "SYN", + "-j", "TCPMSS", + "--clamp-mss-to-pmtu", + "-m", "comment", "--comment", + fmt.Sprintf("vprox ipip TCP MSS outbound rule for %s", srv.Ifname()), + } + in := []string{ + "-i", srv.ipipIfaceWildcard(), + "-p", "tcp", + "--tcp-flags", "SYN,RST", "SYN", + "-j", "TCPMSS", + "--clamp-mss-to-pmtu", + "-m", "comment", "--comment", + fmt.Sprintf("vprox ipip TCP MSS inbound rule for %s", srv.Ifname()), + } + if enabled { + if err := srv.Ipt.AppendUnique("mangle", "FORWARD", out...); err != nil { + return fmt.Errorf("append ipip outbound MSS rule: %v", err) + } + if err := srv.Ipt.AppendUnique("mangle", "FORWARD", in...); err != nil { + return fmt.Errorf("append ipip inbound MSS rule: %v", err) + } + return nil + } + + // Cleanup path: attempt both deletions independently so a failure on + // the first doesn't leak the second (matches the WG MSS cleanup + // pattern in CleanupIptables). + if err := srv.Ipt.Delete("mangle", "FORWARD", out...); err != nil { + log.Printf("failed to remove ipip outbound MSS rule: %v", err) + } + if err := srv.Ipt.Delete("mangle", "FORWARD", in...); err != nil { + log.Printf("failed to remove ipip inbound MSS rule: %v", err) + } + return nil +} diff --git a/lib/ipip_test.go b/lib/ipip_test.go new file mode 100644 index 0000000..e63e9e0 --- /dev/null +++ b/lib/ipip_test.go @@ -0,0 +1,119 @@ +package lib + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testWgCidr builds a WgCidr the way server_manager.go does -- PrefixFrom on +// the network base + 1 (the server's reserved IP) -- so its .Addr() matches +// a real running server's rather than the masked network address. +func testWgCidr(cidr string) netip.Prefix { + p := netip.MustParsePrefix(cidr) + return netip.PrefixFrom(p.Masked().Addr().Next(), p.Bits()) +} + +func TestIpipIfname(t *testing.T) { + srv := &Server{Index: 0, WgCidr: testWgCidr("10.100.0.0/16")} + name, err := srv.ipipIfname(netip.AddrFrom4([4]byte{10, 100, 0, 2})) + require.NoError(t, err) + assert.Equal(t, "vp0-1", name) + + name, err = srv.ipipIfname(netip.AddrFrom4([4]byte{10, 100, 1, 3})) + require.NoError(t, err) + assert.Equal(t, "vp0-258", name) + + srv = &Server{Index: 7, WgCidr: testWgCidr("10.100.0.0/16")} + name, err = srv.ipipIfname(netip.AddrFrom4([4]byte{10, 100, 255, 255})) + require.NoError(t, err) + assert.Equal(t, "vp7-65534", name) +} + +// TestIpipIfnameWithinIfnamsiz locks in the reasoning from ipipIfname's doc +// comment: the "vp-" name must stay within IFNAMSIZ (15 +// visible chars) even at the largest possible index and offset for any +// realistic CIDR width. +func TestIpipIfnameWithinIfnamsiz(t *testing.T) { + srv := &Server{Index: 65535, WgCidr: testWgCidr("10.100.0.0/16")} + name, err := srv.ipipIfname(netip.AddrFrom4([4]byte{10, 100, 255, 255})) + require.NoError(t, err) + assert.Equal(t, "vp65535-65534", name) + assert.LessOrEqual(t, len(name), 15, "interface name exceeds IFNAMSIZ") +} + +// TestIpipIfnameNoCollisionForWideCidr regression-tests the bug where two +// distinct peers inside a CIDR wider than /16 would produce the same +// interface name because only the low 16 bits of peerIP were used. +func TestIpipIfnameNoCollisionForWideCidr(t *testing.T) { + srv := &Server{Index: 0, WgCidr: testWgCidr("10.0.0.0/8")} + a, err := srv.ipipIfname(netip.AddrFrom4([4]byte{10, 0, 0, 2})) + require.NoError(t, err) + b, err := srv.ipipIfname(netip.AddrFrom4([4]byte{10, 1, 0, 2})) + require.NoError(t, err) + assert.NotEqual(t, a, b, "names must differ for distinct peers in the same CIDR") +} + +// TestIpipIfnameRejectsOversizedNames verifies the hard IFNAMSIZ guard so an +// extreme CIDR + server index combination fails loudly instead of silently +// truncating or colliding. +func TestIpipIfnameRejectsOversizedNames(t *testing.T) { + // A /4 CIDR (28 host bits) plus the maximum server index yields a + // 9-digit offset on top of "vp65535-" (8 chars): 8 + 9 = 17 > 15, so + // this must error. + srv := &Server{Index: 65535, WgCidr: testWgCidr("0.0.0.0/4")} + _, err := srv.ipipIfname(netip.AddrFrom4([4]byte{15, 255, 255, 255})) + assert.Error(t, err, "expected error for ifname exceeding IFNAMSIZ") +} + +// TestIpipPeerFromIfname verifies the ifname parser used by the startup +// sweep round-trips with ipipIfname and rejects names that don't belong to +// this server's IPIP interfaces. +func TestIpipPeerFromIfname(t *testing.T) { + srv := &Server{Index: 7, WgCidr: testWgCidr("10.100.0.0/16")} + for _, peer := range []netip.Addr{ + netip.AddrFrom4([4]byte{10, 100, 0, 2}), + netip.AddrFrom4([4]byte{10, 100, 1, 3}), + netip.AddrFrom4([4]byte{10, 100, 255, 255}), + } { + name, err := srv.ipipIfname(peer) + require.NoError(t, err) + got, ok := srv.ipipPeerFromIfname(name) + require.True(t, ok, "expected %s to parse", name) + assert.Equal(t, peer, got) + } + + for _, name := range []string{"vp8-1", "eth0", "vp7-", "vp7-x", "vproxy7"} { + _, ok := srv.ipipPeerFromIfname(name) + assert.False(t, ok, "expected %s to be rejected", name) + } +} + +func TestIpipIfaceWildcard(t *testing.T) { + assert.Equal(t, "vp0-+", (&Server{Index: 0}).ipipIfaceWildcard()) + assert.Equal(t, "vp42-+", (&Server{Index: 42}).ipipIfaceWildcard()) +} + +func TestIpipPeerAcceptRule(t *testing.T) { + peerIP := netip.AddrFrom4([4]byte{10, 100, 0, 5}) + rule := ipipPeerAcceptRule("vp0-5", peerIP) + assert.Equal(t, []string{ + "-i", "vp0-5", + "-s", "10.100.0.5/32", + "-j", "ACCEPT", + "-m", "comment", "--comment", + "vprox ipip accept peer 10.100.0.5 on vp0-5", + }, rule) +} + +func TestIpipPeerDropRule(t *testing.T) { + rule := ipipPeerDropRule("vp0-5") + assert.Equal(t, []string{ + "-i", "vp0-5", + "-j", "DROP", + "-m", "comment", "--comment", + "vprox ipip drop spoofed on vp0-5", + }, rule) +} diff --git a/lib/server.go b/lib/server.go index e775fc3..844432a 100644 --- a/lib/server.go +++ b/lib/server.go @@ -93,7 +93,7 @@ type Server struct { ipAllocator *IpAllocator - mu sync.Mutex // Protects the fields below. + mu sync.Mutex // Protects newPeers and peerIPs. // newPeers records the time of each peer's most recent /connect, granting // a grace period during which the idle reaper won't remove the peer. This // protects peers that haven't completed a WireGuard handshake yet, and @@ -107,6 +107,22 @@ type Server struct { // kernel device: an entry is added when a peer is allocated an IP and // removed when the reaper deletes an idle peer. peerIPs map[wgtypes.Key]netip.Addr + + // ipipMu protects ipipPeers and ipipClosed. It is separate from mu so + // that the IPIP peer bookkeeping does not contend with the WireGuard + // peer state. + ipipMu sync.Mutex + ipipPeers map[netip.Addr]*ipipPeer + // ipipClosed is set by CleanupIpip so that a /connect-ipip handler + // that outlives the shutdown drain cannot register a new tunnel after + // cleanup has already run. + ipipClosed bool + // ipipCreateMu serializes the /connect-ipip create path. The kernel + // allows only one IPIP tunnel per (local, remote) pair, so two + // parallel requests from the same client would both miss the peer + // map and the loser's LinkAdd would fail with EEXIST; serializing + // creation lets the second request reuse the winner's tunnel. + ipipCreateMu sync.Mutex } // InitState initializes the private server state. @@ -132,8 +148,25 @@ func (srv *Server) InitState() error { if reservedIp != srv.WgCidr.Addr() { return fmt.Errorf("reserved IP address mistamches CIDR: %v != %v", reservedIp, srv.WgCidr.Addr()) } + + // Fail fast if WgCidr is wide enough that a peer's IPIP interface name + // would exceed IFNAMSIZ; otherwise every /connect-ipip request would + // 500 at runtime instead. The longest name comes from the last address + // of the block. + maskedWg := srv.WgCidr.Masked() + wgBase := maskedWg.Addr().As4() + lastInt := (uint32(wgBase[0])<<24 | uint32(wgBase[1])<<16 | + uint32(wgBase[2])<<8 | uint32(wgBase[3])) | (^uint32(0) >> uint(maskedWg.Bits())) + lastAddr := netip.AddrFrom4([4]byte{ + byte(lastInt >> 24), byte(lastInt >> 16), byte(lastInt >> 8), byte(lastInt), + }) + if _, err := srv.ipipIfname(lastAddr); err != nil { + return fmt.Errorf("WgCidr %v too wide for IPIP: %v", srv.WgCidr, err) + } + srv.newPeers = make(map[wgtypes.Key]time.Time) srv.peerIPs = make(map[wgtypes.Key]netip.Addr) + srv.ipipPeers = make(map[netip.Addr]*ipipPeer) return nil } @@ -620,6 +653,15 @@ func (srv *Server) StartIptables() error { return fmt.Errorf("failed to add inbound TCP MSS rule: %v", err) } + // Wildcard TCP MSS clamping for this server's IPIP interfaces. The + // interfaces themselves are created lazily by /connect-ipip; the + // FORWARD ACCEPT/DROP filter is installed per peer at that point so + // each tunnel only accepts traffic with the inner source IP we + // assigned to it (see addIpipPeerFilter). + if err := srv.iptablesIpipMssRules(true); err != nil { + return fmt.Errorf("failed to add ipip MSS rules: %v", err) + } + // SNAT rule for internal network traffic. This is currently only applicable for boxes in // the US. if srv.Region == "us-west" { @@ -719,6 +761,10 @@ func (srv *Server) CleanupIptables() { log.Printf("failed to remove inbound TCP MSS rule: %v", err) } + if err := srv.iptablesIpipMssRules(false); err != nil { + log.Printf("failed to remove ipip MSS rules: %v", err) + } + if srv.Region == "us-west" { // Remove SNAT rule for internal traffic rule = []string{ @@ -866,12 +912,37 @@ func (srv *Server) addBindAddr() error { }) } +// requireExternalRemote rejects requests whose source IP is inside +// srv.WgCidr -- i.e. requests that arrived from inside a VPN tunnel rather +// than from the public network. The control plane is meant to be reached +// from clients' outer addresses only; both connectHandler and +// connectIpipHandler treat r.RemoteAddr as identity-relevant input (for +// logging and for the IPIP peer-map key respectively), so a request whose +// apparent source is a VPN-internal IP is either spoofed or a +// misconfigured client routing everything into the tunnel. Either way it +// is not legitimate, and rejecting it here keeps the assumption that +// "RemoteAddr is the client's real outer address" honest for every +// control-plane handler without each having to recheck. +func (srv *Server) requireExternalRemote(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err == nil { + if ip, err := netip.ParseAddr(host); err == nil && srv.WgCidr.Contains(ip) { + http.Error(w, "control-plane requests must originate outside the VPN", http.StatusForbidden) + return + } + } + h(w, r) + } +} + func (srv *Server) ListenForHttps() error { if !srv.BindAddr.Is4() { return fmt.Errorf("invalid IPv4 bind address: %v", srv.BindAddr) } go srv.removeIdlePeersLoop() + go srv.removeIdleIpipPeersLoop() // Some bind addresses may not have been added to the network interface. If // that is the case, we need to add it (transiently). @@ -880,7 +951,8 @@ func (srv *Server) ListenForHttps() error { mux := http.NewServeMux() mux.HandleFunc("/", srv.indexHandler) - mux.HandleFunc("/connect", srv.connectHandler) + mux.HandleFunc("/connect", srv.requireExternalRemote(srv.connectHandler)) + mux.HandleFunc("/connect-ipip", srv.requireExternalRemote(srv.connectIpipHandler)) cert, err := loadServerTls() if err != nil { @@ -913,7 +985,14 @@ func (srv *Server) ListenForHttps() error { select { case <-srv.Ctx.Done(): log.Printf("server no longer listening on %v:443\n", srv.BindAddr) - return httpServer.Shutdown(srv.Ctx) + // srv.Ctx is already cancelled here, so passing it to Shutdown + // would return immediately without draining in-flight handlers. + // Drain with a fresh deadline so handlers (notably /connect-ipip, + // whose tunnels are torn down right after this function returns) + // finish before cleanup runs. + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return httpServer.Shutdown(shutdownCtx) case err = <-errCh: return err } diff --git a/lib/server_manager.go b/lib/server_manager.go index 9218518..6f9594e 100644 --- a/lib/server_manager.go +++ b/lib/server_manager.go @@ -151,6 +151,16 @@ func (sm *ServerManager) Start(ip netip.Addr) error { log.Printf("[%v] failed to start iptables: %v", ip, err) return } + // IPIP tunnels have no kernel-adoption path yet (unlike WireGuard, + // which RestorePeersFromKernel re-adopts), so tear them down on + // shutdown to keep the allocator and kernel state consistent, and + // sweep any tunnels a crashed predecessor left behind before their + // stale /32 routes can blackhole freshly allocated peer IPs. + defer srv.CleanupIpip() + if err := srv.SweepStaleIpip(); err != nil { + log.Printf("[%v] failed to sweep stale ipip tunnels: %v", ip, err) + return + } if err := srv.ListenForHttps(); err != nil { log.Printf("[%v] https server failed: %v", ip, err)