From 049ee94981757b366e85de6c1f9690896c2163d0 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 7 Aug 2026 17:44:42 +0200 Subject: [PATCH 01/42] proxy: scope datapath rules to the node hosting the backend Every node programmed the svc_pod/pod_svc maps and the port-filter sets for every service, so a node that did not host the backend still rewrote the destination of a packet leaving it. That premature DNAT breaks conntrack on the owning node. It records the flow as (srcSvcIP -> podIP) because the destination was already translated upstream, while the reply leaves the pod and gets its source rewritten to the service IP by egress_snat, which runs at prerouting priority raw, before conntrack. The tuple (dstSvcIP -> srcSvcIP) matches nothing, the reply is not established, and port_filter drops it since the initiator's ephemeral port is not in its own allowed_ports. The visible effect is that a PortList (wholeIP=false) backend can no longer open a connection to another cozy-proxy managed backend on a different node: the SYN arrives, the SYN-ACK is generated and dropped, and the caller hangs. Traffic from outside the cluster is unaffected, because it is not translated before reaching the owning node. Program the rules only where the backend pod runs, keyed on the endpoint's NodeName against NODE_NAME. Every node then sees a consistent conntrack view, and the maps only carry local entries. Rules for a pod that moved away are withdrawn, both on endpoint events and by the startup cleanup, so state inherited from a cluster-wide build is purged on upgrade. NODE_NAME is read from the environment; when it is absent the check is disabled and the previous cluster-wide behavior is kept, so the binary still runs under a chart that does not inject it yet. Signed-off-by: Mattia Eleuteri --- main.go | 11 ++ pkg/controllers/services_controller.go | 114 +++++++++++++++--- pkg/controllers/services_controller_test.go | 127 ++++++++++++++++++++ 3 files changed, 233 insertions(+), 19 deletions(-) diff --git a/main.go b/main.go index 5d6f9dd..e870daa 100644 --- a/main.go +++ b/main.go @@ -68,9 +68,20 @@ func main() { os.Exit(1) } + // Datapath rules are programmed only for backend pods hosted on this node. + // When NODE_NAME is absent the check is disabled and every service is + // programmed, which is the pre-node-local behavior: degraded, but it keeps + // a new binary working under a chart that does not inject the variable yet. + nodeName := os.Getenv("NODE_NAME") + if nodeName == "" { + log.Info("NODE_NAME is not set, falling back to programming rules for every node's backends; " + + "set it from spec.nodeName to scope rules to this node") + } + controller := &controllers.ServicesController{ Clientset: clientset, Proxy: &proxy.NFTProxyProcessor{}, + NodeName: nodeName, } if err := mgr.Add(controller); err != nil { diff --git a/pkg/controllers/services_controller.go b/pkg/controllers/services_controller.go index cb00a46..88e174a 100644 --- a/pkg/controllers/services_controller.go +++ b/pkg/controllers/services_controller.go @@ -96,6 +96,83 @@ type ServicesController struct { Clientset *kubernetes.Clientset Services *ServiceMap Proxy nat.ProxyProcessor + + // NodeName is the node this instance runs on. Datapath rules are only + // programmed for backend pods hosted here. Empty disables the check and + // restores the previous cluster-wide behavior. + NodeName string +} + +// endpointNode returns the node hosting the endpoint's first address. +func endpointNode(ep *v1.Endpoints) (string, bool) { + if !hasValidEndpointIP(ep) { + return "", false + } + node := ep.Subsets[0].Addresses[0].NodeName + if node == nil || *node == "" { + return "", false + } + return *node, true +} + +// servesEndpoint reports whether this node must program datapath rules for ep. +// +// The rules are node-local: only the node hosting the backend pod may rewrite +// the service IP. Programming them cluster-wide makes a non-owning node +// translate the destination before the packet even leaves it, so the owning +// node records a conntrack tuple the SNATed reply can no longer match, and +// port_filter drops that reply. +// +// When the node name is unknown (NODE_NAME not injected, or an endpoint +// carrying no NodeName) the rules are programmed anyway, so an older chart +// keeps the previous behavior instead of silently losing the datapath. +func (c *ServicesController) servesEndpoint(ep *v1.Endpoints) bool { + if c.NodeName == "" { + return true + } + node, ok := endpointNode(ep) + if !ok { + return true + } + return node == c.NodeName +} + +// applyRules programs the datapath for a (service, endpoint) pair when this +// node hosts the backend pod, and withdraws it otherwise. Both objects must +// already have been checked with hasValidServiceIP/hasValidEndpointIP. +func (c *ServicesController) applyRules(svc *v1.Service, ep *v1.Endpoints, ctx string) { + svcIP := svc.Status.LoadBalancer.Ingress[0].IP + podIP := ep.Subsets[0].Addresses[0].IP + + if !c.servesEndpoint(ep) { + c.withdrawRules(svcIP, podIP, ctx+" (backend not on this node)") + return + } + + c.Proxy.EnsureRules(svcIP, podIP) + c.reconcilePortFilter(svc, svcIP, podIP, ctx) +} + +// withdrawRules removes every datapath entry for the pair. Absent entries are +// not an error. +func (c *ServicesController) withdrawRules(svcIP, podIP, ctx string) { + c.clearPortFilter(svcIP, podIP, ctx) + c.Proxy.DeleteRules(svcIP, podIP) +} + +// withdrawStaleEndpoint drops the rules of a previous endpoint whose pod IP no +// longer matches the current one, which is what happens when a VM is migrated +// to another node. Without this the old node keeps a mapping for a pod it no +// longer hosts. +func (c *ServicesController) withdrawStaleEndpoint(svc *v1.Service, prev *v1.Endpoints, podIP, ctx string) { + if !hasValidServiceIP(svc) || !hasValidEndpointIP(prev) { + return + } + prevPodIP := prev.Subsets[0].Addresses[0].IP + if prevPodIP == podIP { + return + } + c.withdrawRules(svc.Status.LoadBalancer.Ingress[0].IP, prevPodIP, ctx+" (stale endpoint)") } // Start initializes the NAT, runs the service and endpoint informers, and cleans up removed services. @@ -231,11 +308,7 @@ func (c *ServicesController) addServiceFunc(obj interface{}) { if err == nil && ep != nil && hasValidEndpointIP(ep) && hasValidServiceIP(svc) { se.Endpoint = ep c.Services.Set(svc.Namespace, svc.Name, se) - svcIP := svc.Status.LoadBalancer.Ingress[0].IP - podIP := ep.Subsets[0].Addresses[0].IP - // Ensure NAT mapping rules are set. - c.Proxy.EnsureRules(svcIP, podIP) - c.reconcilePortFilter(svc, svcIP, podIP, "on svc add") + c.applyRules(svc, ep, "on svc add") } } @@ -330,10 +403,10 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) { // At this point, both the Service and Endpoint have valid IPs. // Ensure NAT mapping is up-to-date. - svcIP := svc.Status.LoadBalancer.Ingress[0].IP - podIP := ep.Subsets[0].Addresses[0].IP - c.Proxy.EnsureRules(svcIP, podIP) - c.reconcilePortFilter(svc, svcIP, podIP, "on svc update") + if se, exists := c.Services.Get(svc.Namespace, svc.Name); exists { + c.withdrawStaleEndpoint(svc, se.Endpoint, ep.Subsets[0].Addresses[0].IP, "on svc update") + } + c.applyRules(svc, ep, "on svc update") // Update or add the service mapping with the new endpoint. c.Services.Set(svc.Namespace, svc.Name, &ServiceEndpoints{Service: svc, Endpoint: ep}) @@ -360,10 +433,8 @@ func (c *ServicesController) addEndpointFunc(obj interface{}) { // If both the Service and the Endpoint have valid IPs, ensure NAT mapping rules. if hasValidServiceIP(se.Service) && hasValidEndpointIP(ep) { - svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP - podIP := ep.Subsets[0].Addresses[0].IP - c.Proxy.EnsureRules(svcIP, podIP) - c.reconcilePortFilter(se.Service, svcIP, podIP, "on endpoint add") + c.withdrawStaleEndpoint(se.Service, se.Endpoint, ep.Subsets[0].Addresses[0].IP, "on endpoint add") + c.applyRules(se.Service, ep, "on endpoint add") } } @@ -420,10 +491,8 @@ func (c *ServicesController) updateEndpointFunc(oldObj, newObj interface{}) { if !hasValidEndpointIP(ep) { return } - svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP - podIP := ep.Subsets[0].Addresses[0].IP - c.Proxy.EnsureRules(svcIP, podIP) - c.reconcilePortFilter(se.Service, svcIP, podIP, "on endpoint update") + c.withdrawStaleEndpoint(se.Service, se.Endpoint, ep.Subsets[0].Addresses[0].IP, "on endpoint update") + c.applyRules(se.Service, ep, "on endpoint update") c.Services.SetEndpoint(ep.Namespace, ep.Name, ep) } @@ -555,6 +624,13 @@ func (c *ServicesController) cleanupRemovedServices() error { if serviceEndpoints.Service != nil && serviceEndpoints.Endpoint != nil { var serviceIP, endpointIP string + // Backends hosted elsewhere are not ours to program, so they must + // not be kept: this is what purges entries inherited from a build + // that programmed every service on every node. + if !c.servesEndpoint(serviceEndpoints.Endpoint) { + continue + } + if len(serviceEndpoints.Service.Status.LoadBalancer.Ingress) > 0 { serviceIP = serviceEndpoints.Service.Status.LoadBalancer.Ingress[0].IP } @@ -582,7 +658,7 @@ func (c *ServicesController) cleanupRemovedServices() error { if !hasValidServiceIP(se.Service) || !hasValidEndpointIP(se.Endpoint) { continue } - if wholeIPPassthrough(se.Service) { + if wholeIPPassthrough(se.Service) || !c.servesEndpoint(se.Endpoint) { continue } keepFilters[se.Service.Status.LoadBalancer.Ingress[0].IP] = nat.PortFilterEntry{ @@ -603,7 +679,7 @@ func (c *ServicesController) cleanupRemovedServices() error { if !hasValidServiceIP(se.Service) || !hasValidEndpointIP(se.Endpoint) { continue } - if wholeIPPassthrough(se.Service) || !allowICMP(se.Service) { + if wholeIPPassthrough(se.Service) || !allowICMP(se.Service) || !c.servesEndpoint(se.Endpoint) { continue } keepICMP[se.Service.Status.LoadBalancer.Ingress[0].IP] = se.Endpoint.Subsets[0].Addresses[0].IP diff --git a/pkg/controllers/services_controller_test.go b/pkg/controllers/services_controller_test.go index 549e4b4..40aae84 100644 --- a/pkg/controllers/services_controller_test.go +++ b/pkg/controllers/services_controller_test.go @@ -5,8 +5,135 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nat "github.com/cozystack/cozy-proxy/pkg/proxy" ) +// epOnNode builds an Endpoints with a single address, optionally pinned to a +// node. Pass an empty node to omit NodeName, as an endpoint with no scheduling +// information would have. +func epOnNode(podIP, node string) *v1.Endpoints { + addr := v1.EndpointAddress{IP: podIP} + if node != "" { + addr.NodeName = &node + } + return &v1.Endpoints{ + Subsets: []v1.EndpointSubset{{Addresses: []v1.EndpointAddress{addr}}}, + } +} + +// lbService builds a LoadBalancer service already assigned the given IP. +func lbService(svcIP string, annot map[string]string) *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Annotations: annot}, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{{IP: svcIP}}, + }, + }, + } +} + +// recordingProxy captures which datapath calls a controller makes. +type recordingProxy struct { + nat.DummyProxyProcessor + calls []string +} + +func (r *recordingProxy) EnsureRules(svcIP, podIP string) error { + r.calls = append(r.calls, "EnsureRules") + return nil +} + +func (r *recordingProxy) DeleteRules(svcIP, podIP string) error { + r.calls = append(r.calls, "DeleteRules") + return nil +} + +func (r *recordingProxy) EnsurePortFilter(svcIP, podIP string, ports []v1.ServicePort) error { + r.calls = append(r.calls, "EnsurePortFilter") + return nil +} + +func (r *recordingProxy) DeletePortFilter(svcIP, podIP string) error { + r.calls = append(r.calls, "DeletePortFilter") + return nil +} + +func (r *recordingProxy) has(call string) bool { + for _, c := range r.calls { + if c == call { + return true + } + } + return false +} + +func TestServesEndpoint(t *testing.T) { + cases := []struct { + name string + nodeName string + ep *v1.Endpoints + expect bool + }{ + {"backend on this node", "node-a", epOnNode("10.0.0.1", "node-a"), true}, + {"backend on another node", "node-a", epOnNode("10.0.0.1", "node-b"), false}, + {"endpoint without NodeName is programmed", "node-a", epOnNode("10.0.0.1", ""), true}, + {"NODE_NAME unset programs everything", "", epOnNode("10.0.0.1", "node-b"), true}, + {"invalid endpoint is programmed", "node-a", &v1.Endpoints{}, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ctrl := &ServicesController{NodeName: c.nodeName} + if got := ctrl.servesEndpoint(c.ep); got != c.expect { + t.Errorf("servesEndpoint = %v, want %v", got, c.expect) + } + }) + } +} + +// A non-owning node must not translate the service IP: doing so desynchronises +// conntrack on the owning node and gets the reply dropped by port_filter. +func TestApplyRulesSkipsRemoteBackend(t *testing.T) { + svc := lbService("192.0.2.10", map[string]string{"networking.cozystack.io/wholeIP": "false"}) + + local := &recordingProxy{} + localCtrl := &ServicesController{Proxy: local, NodeName: "node-a"} + localCtrl.applyRules(svc, epOnNode("10.0.0.1", "node-a"), "test") + if !local.has("EnsureRules") || !local.has("EnsurePortFilter") { + t.Errorf("owning node must program the datapath, got %v", local.calls) + } + + remote := &recordingProxy{} + remoteCtrl := &ServicesController{Proxy: remote, NodeName: "node-a"} + remoteCtrl.applyRules(svc, epOnNode("10.0.0.1", "node-b"), "test") + if remote.has("EnsureRules") || remote.has("EnsurePortFilter") { + t.Errorf("non-owning node must not program the datapath, got %v", remote.calls) + } + if !remote.has("DeleteRules") { + t.Errorf("non-owning node must withdraw any inherited rules, got %v", remote.calls) + } +} + +// A migrated VM leaves its old mapping behind on the node it came from. +func TestWithdrawStaleEndpoint(t *testing.T) { + svc := lbService("192.0.2.10", map[string]string{"networking.cozystack.io/wholeIP": "false"}) + + moved := &recordingProxy{} + movedCtrl := &ServicesController{Proxy: moved, NodeName: "node-a"} + movedCtrl.withdrawStaleEndpoint(svc, epOnNode("10.0.0.1", "node-a"), "10.0.0.2", "test") + if !moved.has("DeleteRules") { + t.Errorf("changed pod IP must withdraw the previous mapping, got %v", moved.calls) + } + + same := &recordingProxy{} + sameCtrl := &ServicesController{Proxy: same, NodeName: "node-a"} + sameCtrl.withdrawStaleEndpoint(svc, epOnNode("10.0.0.1", "node-a"), "10.0.0.1", "test") + if len(same.calls) != 0 { + t.Errorf("unchanged pod IP must not touch the datapath, got %v", same.calls) + } +} + func svcWith(annot map[string]string) *v1.Service { return &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: annot}} } From 8aee75fcd5761c22427106058f35936d3a61ee92 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 7 Aug 2026 18:11:06 +0200 Subject: [PATCH 02/42] proxy: do not abort the pod when startup cleanup hits ENOENT CleanupRules queued deletions and additions into a single batch and treated any flush error as fatal. Deleting a set element that is already gone reports ENOENT, which fails the whole flush, so the controller returned an error, the manager exited, and the DaemonSet pod entered CrashLoopBackOff with the node's datapath left half-programmed. Scoping the rules to the local node made this reliable rather than rare: the first startup after the change deletes every entry the node inherited for backends it does not host, which is most of them. Commit deletions separately from additions, tolerate ENOENT on the flush the way DeleteRules, DeletePortFilter and DeleteICMPAllow already do, and apply the same split to CleanupPortFilters and CleanupICMPAllow. A cleanup failure is now logged instead of aborting Start, since the informers converge on the next event anyway and staying up with stale entries beats exiting with a partial ruleset. Observed on a 3-node cluster carrying 15 managed services: the transition logs "Ignoring ENOENT on flush" for the cleanup deletions and completes with zero restarts, where it previously crash-looped. Signed-off-by: Mattia Eleuteri --- pkg/controllers/services_controller.go | 9 +++-- pkg/proxy/nft.go | 47 +++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/pkg/controllers/services_controller.go b/pkg/controllers/services_controller.go index 88e174a..a38c1ae 100644 --- a/pkg/controllers/services_controller.go +++ b/pkg/controllers/services_controller.go @@ -267,12 +267,15 @@ func (c *ServicesController) Start(ctx context.Context) error { } log.Info("endpoints synchronization completed") - // Run cleanup for removed services. + // Run cleanup for removed services. A failure here is logged but does not + // abort: exiting takes the pod down and leaves the node's datapath + // half-programmed, whereas the informers below converge on the next event. log.Info("running cleanup for removed services") if err := c.cleanupRemovedServices(); err != nil { - return fmt.Errorf("failed to cleanup removed services: %w", err) + log.Error(err, "cleanup of removed services failed, continuing with reconciliation") + } else { + log.Info("cleanup of removed services completed") } - log.Info("cleanup of removed services completed") <-ctx.Done() log.Info("shutting down services-controller") diff --git a/pkg/proxy/nft.go b/pkg/proxy/nft.go index afe72ac..1caa33f 100644 --- a/pkg/proxy/nft.go +++ b/pkg/proxy/nft.go @@ -527,6 +527,24 @@ func (p *NFTProxyProcessor) DeleteRules(svcIP, podIP string) error { return nil } +// flushTolerateENOENT commits the pending batch and treats ENOENT as success. +// +// Deleting a set element that is already gone reports ENOENT, which fails the +// whole flush. Deletions must therefore be committed on their own, so a stale +// element cannot mask a genuine failure among the additions that would +// otherwise share the batch. +func (p *NFTProxyProcessor) flushTolerateENOENT(op string) error { + err := p.conn.Flush() + if err == nil { + return nil + } + if errors.Is(err, unix.ENOENT) { + log.Info("Ignoring ENOENT on flush — element already gone", "op", op) + return nil + } + return err +} + // CleanupRules receives a keepMap (keys: svcIP, values: podIP) representing the desired state. // It recovers from an inconsistent state by: // 1. Removing any mappings in the pod_svc and svc_pod maps that do not match keepMap. @@ -585,6 +603,13 @@ func (p *NFTProxyProcessor) CleanupRules(keepMap map[string]string) error { log.Error(err, "Failed to delete inconsistent mappings from svc_pod") return fmt.Errorf("failed to delete inconsistent mappings from svc_pod: %v", err) } + // Commit the deletions before queueing the additions below: an element + // that is already gone fails the flush, and a shared batch would report + // that as a cleanup failure, which aborts the controller at startup. + if err := p.flushTolerateENOENT("CleanupRules deletions"); err != nil { + log.Error(err, "Failed to commit cleanup deletions") + return fmt.Errorf("failed to commit cleanup deletions: %v", err) + } log.Info("Inconsistent mappings removed from both maps") } else { log.Info("No inconsistent mappings found in maps") @@ -617,7 +642,10 @@ func (p *NFTProxyProcessor) CleanupRules(keepMap map[string]string) error { } // --- Final commit --- - if err := p.conn.Flush(); err != nil { + // Startup cleanup must not be fatal: aborting here takes the whole + // DaemonSet pod down and leaves the node's datapath half-programmed, while + // the reconcile loop would have converged on the next event anyway. + if err := p.flushTolerateENOENT("CleanupRules additions"); err != nil { log.Error(err, "Failed to commit cleanup changes") return fmt.Errorf("failed to commit cleanup changes: %v", err) } @@ -804,6 +832,13 @@ func (p *NFTProxyProcessor) CleanupPortFilters(keep map[string]PortFilterEntry) return fmt.Errorf("failed to delete stale allowed_ports: %v", err) } } + // Commit deletions separately so an element that is already gone cannot + // fail the batch carrying the additions below. + if len(delPods) > 0 || len(delPorts) > 0 { + if err := p.flushTolerateENOENT("CleanupPortFilters deletions"); err != nil { + return fmt.Errorf("failed to flush CleanupPortFilters deletions: %v", err) + } + } if len(addPods) > 0 { if err := p.conn.SetAddElements(p.filteredPods, addPods); err != nil { return fmt.Errorf("failed to add filtered_pods: %v", err) @@ -815,8 +850,8 @@ func (p *NFTProxyProcessor) CleanupPortFilters(keep map[string]PortFilterEntry) } } - // 5. Single flush. - if err := p.conn.Flush(); err != nil { + // 5. Commit the additions. Startup cleanup must not abort the pod. + if err := p.flushTolerateENOENT("CleanupPortFilters additions"); err != nil { return fmt.Errorf("failed to flush CleanupPortFilters: %v", err) } log.Info("CleanupPortFilters completed", @@ -948,6 +983,10 @@ func (p *NFTProxyProcessor) CleanupICMPAllow(keep map[string]string) error { if err := p.conn.SetDeleteElements(p.icmpAllowedPods, delPods); err != nil { return fmt.Errorf("failed to delete stale icmp_allowed_pods: %v", err) } + // Commit deletions separately: see flushTolerateENOENT. + if err := p.flushTolerateENOENT("CleanupICMPAllow deletions"); err != nil { + return fmt.Errorf("failed to flush CleanupICMPAllow deletions: %v", err) + } } if len(addPods) > 0 { if err := p.conn.SetAddElements(p.icmpAllowedPods, addPods); err != nil { @@ -955,7 +994,7 @@ func (p *NFTProxyProcessor) CleanupICMPAllow(keep map[string]string) error { } } - if err := p.conn.Flush(); err != nil { + if err := p.flushTolerateENOENT("CleanupICMPAllow additions"); err != nil { return fmt.Errorf("failed to flush CleanupICMPAllow: %v", err) } log.Info("CleanupICMPAllow completed", From 16057043f51112d639047a760ba2fb98048d3f3d Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 15 Sep 2026 12:05:23 +0200 Subject: [PATCH 03/42] proxy: keep the egress SNAT map cluster-wide Scoping the datapath to the node hosting the backend was right for the destination rewrite and wrong for the source one. Both were scoped at once, which fixed a managed backend calling another managed backend and broke every other intra-cluster client. An intra-cluster client is source-NATed by kube-ovn to its own node address, which OVN knows how to reach. The backend's reply is therefore tunnelled straight to that node over Geneve and never traverses the hosting node's netfilter hooks, so egress_snat there never sees it. The client's node is the only place left where the pod IP can still be turned back into the service IP, and with pod_svc node-local it holds no entry for a backend it does not host. The reply arrives with saddr=podIP, matches nothing in the client's conntrack, and the client answers with a RST. A connection to a backend on the same node works, one to a backend on any other node hangs. A managed client escapes this because egress_snat gives it its own service IP as source, an address OVN does not know: its peer's reply leaves through the host stack and gets rewritten on the way out. That is why scoping both maps looked correct. Program pod_svc on every node for every managed service, and keep svc_pod, allowed_ports and icmp_allowed_pods on the hosting node only. EnsureRules/DeleteRules are split into the egress and ingress halves so each map resolves its own key conflicts, and CleanupRules now takes one keep set per map, so a node purges the entries of the other scope on upgrade in either direction. Verified on a three-node cluster: adding the missing pod_svc entry by hand on the client's node unblocks the cross-node flow and removing it breaks it again, while a client turned into a managed backend reaches the same peers with no such entry. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 14 + pkg/controllers/services_controller.go | 129 ++++--- pkg/controllers/services_controller_test.go | 121 +++++- pkg/proxy/dummy.go | 22 +- pkg/proxy/interface.go | 36 +- pkg/proxy/nft.go | 384 +++++++++++--------- 6 files changed, 450 insertions(+), 256 deletions(-) diff --git a/README.md b/README.md index 4c497e1..9c8705c 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,20 @@ The nftables ruleset placed in table `ip cozy_proxy` consists of: `allowICMP: "true"` annotation is set, the pod IP is added to `icmp_allowed_pods` and ICMP toward it is accepted before the drop rule. +### Rule scoping + +The two rewrites are deliberately not scoped alike. Every controller instance +watches every Service, but programs: + +| Object | Scope | Why | +|---|---|---| +| `svc_pod` (`ingress_dnat`), `allowed_ports`, `icmp_allowed_pods` | the node hosting the backend pod | A non-hosting node would rewrite the destination before the packet has even left it. The hosting node then records a conntrack tuple of `(client -> podIP)`, while the reply leaves the pod and gets `saddr` rewritten to the service IP by `egress_snat` at priority `raw`, before conntrack. `(svcIP -> client)` matches nothing, so the reply is not `established` and `port_filter` drops it. | +| `pod_svc` (`egress_snat`) | every node | An intra-cluster client is source-NATed by the CNI to its own node address, which the overlay knows how to reach directly. The backend's reply is then tunnelled straight to that node and never traverses the hosting node's netfilter hooks, so the client's node is the only place left where the pod IP can still be turned back into the service IP. Without the entry there, the reply arrives with the wrong source and the client answers with a RST — a cross-node connection that hangs while the same-node one works. | + +`NODE_NAME` carries the node identity. When it is unset the ingress scope +check is disabled and everything is programmed everywhere, so the binary still +runs under a chart that does not inject it. + ## Installation Install controller using Helm-chart: diff --git a/pkg/controllers/services_controller.go b/pkg/controllers/services_controller.go index a38c1ae..e030517 100644 --- a/pkg/controllers/services_controller.go +++ b/pkg/controllers/services_controller.go @@ -97,9 +97,10 @@ type ServicesController struct { Services *ServiceMap Proxy nat.ProxyProcessor - // NodeName is the node this instance runs on. Datapath rules are only - // programmed for backend pods hosted here. Empty disables the check and - // restores the previous cluster-wide behavior. + // NodeName is the node this instance runs on. It scopes the ingress + // half of the datapath — the destination rewrite and the port filter — + // to backend pods hosted here. Empty disables the check and programs + // everything everywhere. NodeName string } @@ -115,17 +116,20 @@ func endpointNode(ep *v1.Endpoints) (string, bool) { return *node, true } -// servesEndpoint reports whether this node must program datapath rules for ep. +// servesEndpoint reports whether this node hosts ep's backend, and may +// therefore program the ingress half of the datapath. // -// The rules are node-local: only the node hosting the backend pod may rewrite -// the service IP. Programming them cluster-wide makes a non-owning node -// translate the destination before the packet even leaves it, so the owning -// node records a conntrack tuple the SNATed reply can no longer match, and -// port_filter drops that reply. +// Only the hosting node may rewrite the destination. Doing it cluster-wide +// makes a non-owning node translate the destination before the packet even +// leaves it, so the owning node records a conntrack tuple the SNATed reply can +// no longer match, and port_filter drops that reply. +// +// This says nothing about the egress half: the source rewrite is programmed on +// every node regardless, see applyRules. // // When the node name is unknown (NODE_NAME not injected, or an endpoint -// carrying no NodeName) the rules are programmed anyway, so an older chart -// keeps the previous behavior instead of silently losing the datapath. +// carrying no NodeName) the ingress rules are programmed anyway, so an older +// chart keeps the previous behavior instead of silently losing the datapath. func (c *ServicesController) servesEndpoint(ep *v1.Endpoints) bool { if c.NodeName == "" { return true @@ -137,27 +141,55 @@ func (c *ServicesController) servesEndpoint(ep *v1.Endpoints) bool { return node == c.NodeName } -// applyRules programs the datapath for a (service, endpoint) pair when this -// node hosts the backend pod, and withdraws it otherwise. Both objects must -// already have been checked with hasValidServiceIP/hasValidEndpointIP. +// applyRules programs the datapath for a (service, endpoint) pair. The two +// halves have different scopes. Both objects must already have been checked +// with hasValidServiceIP/hasValidEndpointIP. func (c *ServicesController) applyRules(svc *v1.Service, ep *v1.Endpoints, ctx string) { svcIP := svc.Status.LoadBalancer.Ingress[0].IP podIP := ep.Subsets[0].Addresses[0].IP + // The source rewrite goes on every node, including the ones that do not + // host the backend. When the client is inside the cluster, kube-ovn SNATs + // it to its own node address, which OVN knows how to reach: the backend's + // reply is then handed straight to that node over the Geneve tunnel and + // never traverses the backend node's netfilter hooks. The client's node is + // the last place where the pod IP can still be turned back into the + // service IP the client's conntrack is waiting for. Without it the reply + // arrives with the wrong source and the client answers it with a RST. + if err := c.Proxy.EnsureEgressSNAT(svcIP, podIP); err != nil { + log.Error(err, "failed to ensure egress SNAT "+ctx, "svcIP", svcIP, "podIP", podIP) + } + + // The destination rewrite and the port filter stay on the hosting node. if !c.servesEndpoint(ep) { - c.withdrawRules(svcIP, podIP, ctx+" (backend not on this node)") + c.withdrawIngressRules(svcIP, podIP, ctx+" (backend not on this node)") return } - c.Proxy.EnsureRules(svcIP, podIP) + if err := c.Proxy.EnsureIngressDNAT(svcIP, podIP); err != nil { + log.Error(err, "failed to ensure ingress DNAT "+ctx, "svcIP", svcIP, "podIP", podIP) + } c.reconcilePortFilter(svc, svcIP, podIP, ctx) } -// withdrawRules removes every datapath entry for the pair. Absent entries are -// not an error. -func (c *ServicesController) withdrawRules(svcIP, podIP, ctx string) { +// withdrawIngressRules removes the ingress half only — destination rewrite and +// port filter — and leaves the source rewrite in place. This is what a node +// that does not host the backend must end up with. +func (c *ServicesController) withdrawIngressRules(svcIP, podIP, ctx string) { c.clearPortFilter(svcIP, podIP, ctx) - c.Proxy.DeleteRules(svcIP, podIP) + if err := c.Proxy.DeleteIngressDNAT(svcIP, podIP); err != nil { + log.Error(err, "failed to delete ingress DNAT "+ctx, "svcIP", svcIP, "podIP", podIP) + } +} + +// withdrawRules removes every datapath entry for the pair, both halves. Used +// when the pair itself is going away: service deleted, endpoint gone, or a pod +// IP that has been replaced. Absent entries are not an error. +func (c *ServicesController) withdrawRules(svcIP, podIP, ctx string) { + c.withdrawIngressRules(svcIP, podIP, ctx) + if err := c.Proxy.DeleteEgressSNAT(svcIP, podIP); err != nil { + log.Error(err, "failed to delete egress SNAT "+ctx, "svcIP", svcIP, "podIP", podIP) + } } // withdrawStaleEndpoint drops the rules of a previous endpoint whose pod IP no @@ -334,8 +366,7 @@ func (c *ServicesController) deleteServiceFunc(obj interface{}) { svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP podIP := se.Endpoint.Subsets[0].Addresses[0].IP - c.clearPortFilter(svcIP, podIP, "on svc deletion") - c.Proxy.DeleteRules(svcIP, podIP) + c.withdrawRules(svcIP, podIP, "on svc deletion") c.Services.Delete(svc.Namespace, svc.Name) } @@ -354,8 +385,7 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) { if hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) { svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP podIP := se.Endpoint.Subsets[0].Addresses[0].IP - c.clearPortFilter(svcIP, podIP, "on annotation removal") - c.Proxy.DeleteRules(svcIP, podIP) + c.withdrawRules(svcIP, podIP, "on annotation removal") } c.Services.Delete(svc.Namespace, svc.Name) } @@ -368,8 +398,7 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) { if hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) { svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP podIP := se.Endpoint.Subsets[0].Addresses[0].IP - c.clearPortFilter(svcIP, podIP, "on svc IP loss") - c.Proxy.DeleteRules(svcIP, podIP) + c.withdrawRules(svcIP, podIP, "on svc IP loss") } c.Services.Delete(svc.Namespace, svc.Name) } @@ -395,9 +424,10 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) { if ep == nil || !hasValidEndpointIP(ep) { if se, exists := c.Services.Get(svc.Namespace, svc.Name); exists && hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) { - c.Proxy.DeleteRules( + c.withdrawRules( se.Service.Status.LoadBalancer.Ingress[0].IP, se.Endpoint.Subsets[0].Addresses[0].IP, + "on endpoint loss", ) } c.Services.Set(svc.Namespace, svc.Name, &ServiceEndpoints{Service: svc, Endpoint: nil}) @@ -459,8 +489,7 @@ func (c *ServicesController) deleteEndpointFunc(obj interface{}) { } svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP podIP := se.Endpoint.Subsets[0].Addresses[0].IP - c.clearPortFilter(svcIP, podIP, "on endpoint delete") - c.Proxy.DeleteRules(svcIP, podIP) + c.withdrawRules(svcIP, podIP, "on endpoint delete") // Set the endpoint to nil. c.Services.SetEndpoint(ep.Namespace, ep.Name, nil) } @@ -482,8 +511,7 @@ func (c *ServicesController) updateEndpointFunc(oldObj, newObj interface{}) { if hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) { svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP oldPodIP := se.Endpoint.Subsets[0].Addresses[0].IP - c.clearPortFilter(svcIP, oldPodIP, "on endpoint invalidation") - c.Proxy.DeleteRules(svcIP, oldPodIP) + c.withdrawRules(svcIP, oldPodIP, "on endpoint invalidation") } c.Services.SetEndpoint(ep.Namespace, ep.Name, ep) return @@ -620,34 +648,33 @@ func (c *ServicesController) clearPortFilter(svcIP, podIP, ctx string) { // cleanupRemovedServices performs an initial cleanup for removed services. func (c *ServicesController) cleanupRemovedServices() error { - keepMap := make(map[string]string) + // keepEgress holds every managed pair, because the source rewrite is + // programmed cluster-wide. keepIngress holds only the pairs whose backend + // runs here, because the destination rewrite is node-local. The difference + // between the two is what purges entries inherited from a build that + // scoped both maps alike. + keepEgress := make(map[string]string) + keepIngress := make(map[string]string) // Get a snapshot of all managed services. allServices := c.Services.GetAll() for _, serviceEndpoints := range allServices { - if serviceEndpoints.Service != nil && serviceEndpoints.Endpoint != nil { - var serviceIP, endpointIP string - - // Backends hosted elsewhere are not ours to program, so they must - // not be kept: this is what purges entries inherited from a build - // that programmed every service on every node. - if !c.servesEndpoint(serviceEndpoints.Endpoint) { - continue - } + if serviceEndpoints.Service == nil || serviceEndpoints.Endpoint == nil { + continue + } + if !hasValidServiceIP(serviceEndpoints.Service) || !hasValidEndpointIP(serviceEndpoints.Endpoint) { + continue + } - if len(serviceEndpoints.Service.Status.LoadBalancer.Ingress) > 0 { - serviceIP = serviceEndpoints.Service.Status.LoadBalancer.Ingress[0].IP - } - if len(serviceEndpoints.Endpoint.Subsets) > 0 && len(serviceEndpoints.Endpoint.Subsets[0].Addresses) > 0 { - endpointIP = serviceEndpoints.Endpoint.Subsets[0].Addresses[0].IP - } + serviceIP := serviceEndpoints.Service.Status.LoadBalancer.Ingress[0].IP + endpointIP := serviceEndpoints.Endpoint.Subsets[0].Addresses[0].IP - if serviceIP != "" && endpointIP != "" { - keepMap[serviceIP] = endpointIP - } + keepEgress[serviceIP] = endpointIP + if c.servesEndpoint(serviceEndpoints.Endpoint) { + keepIngress[serviceIP] = endpointIP } } // Call InitialCleanup with the snapshot. - if err := c.Proxy.CleanupRules(keepMap); err != nil { + if err := c.Proxy.CleanupRules(keepEgress, keepIngress); err != nil { return fmt.Errorf("failed to perform initial cleanup: %w", err) } // Build per-svc port filter snapshot for services in non-passthrough mode. diff --git a/pkg/controllers/services_controller_test.go b/pkg/controllers/services_controller_test.go index 40aae84..46b2ff5 100644 --- a/pkg/controllers/services_controller_test.go +++ b/pkg/controllers/services_controller_test.go @@ -37,16 +37,35 @@ func lbService(svcIP string, annot map[string]string) *v1.Service { // recordingProxy captures which datapath calls a controller makes. type recordingProxy struct { nat.DummyProxyProcessor - calls []string + calls []string + keepEgress map[string]string + keepIngress map[string]string } -func (r *recordingProxy) EnsureRules(svcIP, podIP string) error { - r.calls = append(r.calls, "EnsureRules") +func (r *recordingProxy) EnsureEgressSNAT(svcIP, podIP string) error { + r.calls = append(r.calls, "EnsureEgressSNAT") return nil } -func (r *recordingProxy) DeleteRules(svcIP, podIP string) error { - r.calls = append(r.calls, "DeleteRules") +func (r *recordingProxy) DeleteEgressSNAT(svcIP, podIP string) error { + r.calls = append(r.calls, "DeleteEgressSNAT") + return nil +} + +func (r *recordingProxy) EnsureIngressDNAT(svcIP, podIP string) error { + r.calls = append(r.calls, "EnsureIngressDNAT") + return nil +} + +func (r *recordingProxy) DeleteIngressDNAT(svcIP, podIP string) error { + r.calls = append(r.calls, "DeleteIngressDNAT") + return nil +} + +func (r *recordingProxy) CleanupRules(keepEgress, keepIngress map[string]string) error { + r.calls = append(r.calls, "CleanupRules") + r.keepEgress = keepEgress + r.keepIngress = keepIngress return nil } @@ -92,26 +111,46 @@ func TestServesEndpoint(t *testing.T) { } } -// A non-owning node must not translate the service IP: doing so desynchronises -// conntrack on the owning node and gets the reply dropped by port_filter. -func TestApplyRulesSkipsRemoteBackend(t *testing.T) { +// The owning node programs both halves of the datapath. +func TestApplyRulesProgramsBothHalvesOnOwningNode(t *testing.T) { svc := lbService("192.0.2.10", map[string]string{"networking.cozystack.io/wholeIP": "false"}) local := &recordingProxy{} localCtrl := &ServicesController{Proxy: local, NodeName: "node-a"} localCtrl.applyRules(svc, epOnNode("10.0.0.1", "node-a"), "test") - if !local.has("EnsureRules") || !local.has("EnsurePortFilter") { - t.Errorf("owning node must program the datapath, got %v", local.calls) + + for _, want := range []string{"EnsureEgressSNAT", "EnsureIngressDNAT", "EnsurePortFilter"} { + if !local.has(want) { + t.Errorf("owning node must call %s, got %v", want, local.calls) + } } +} + +// A non-owning node must not translate the destination — doing so +// desynchronises conntrack on the owning node and gets the reply dropped by +// port_filter — but it must still program the source rewrite. +// +// That rewrite is the only thing that repairs a reply reaching this node +// straight over the overlay, which is what happens whenever the client is +// inside the cluster and kube-ovn SNATs it to this node's own address. +func TestApplyRulesKeepsEgressSNATOnRemoteBackend(t *testing.T) { + svc := lbService("192.0.2.10", map[string]string{"networking.cozystack.io/wholeIP": "false"}) remote := &recordingProxy{} remoteCtrl := &ServicesController{Proxy: remote, NodeName: "node-a"} remoteCtrl.applyRules(svc, epOnNode("10.0.0.1", "node-b"), "test") - if remote.has("EnsureRules") || remote.has("EnsurePortFilter") { - t.Errorf("non-owning node must not program the datapath, got %v", remote.calls) + + if !remote.has("EnsureEgressSNAT") { + t.Errorf("non-owning node must still program the source rewrite, got %v", remote.calls) } - if !remote.has("DeleteRules") { - t.Errorf("non-owning node must withdraw any inherited rules, got %v", remote.calls) + if remote.has("DeleteEgressSNAT") { + t.Errorf("non-owning node must not withdraw the source rewrite, got %v", remote.calls) + } + if remote.has("EnsureIngressDNAT") || remote.has("EnsurePortFilter") { + t.Errorf("non-owning node must not program the ingress half, got %v", remote.calls) + } + if !remote.has("DeleteIngressDNAT") { + t.Errorf("non-owning node must withdraw an inherited destination rewrite, got %v", remote.calls) } } @@ -122,8 +161,12 @@ func TestWithdrawStaleEndpoint(t *testing.T) { moved := &recordingProxy{} movedCtrl := &ServicesController{Proxy: moved, NodeName: "node-a"} movedCtrl.withdrawStaleEndpoint(svc, epOnNode("10.0.0.1", "node-a"), "10.0.0.2", "test") - if !moved.has("DeleteRules") { - t.Errorf("changed pod IP must withdraw the previous mapping, got %v", moved.calls) + // A pod IP that is gone must leave nothing behind, in either half: every + // node carries its source rewrite, so every node has to drop it. + for _, want := range []string{"DeleteIngressDNAT", "DeleteEgressSNAT"} { + if !moved.has(want) { + t.Errorf("changed pod IP must call %s, got %v", want, moved.calls) + } } same := &recordingProxy{} @@ -205,3 +248,49 @@ func TestAllowICMP(t *testing.T) { }) } } + +// The startup snapshot must keep every managed pair in the egress map and only +// the locally hosted ones in the ingress map. Handing the same set to both is +// exactly what leaves an intra-cluster client without a usable reply. +func TestCleanupRemovedServicesScopesKeepMaps(t *testing.T) { + annot := map[string]string{"networking.cozystack.io/wholeIP": "false"} + + ctrl := &ServicesController{Proxy: &recordingProxy{}, NodeName: "node-a"} + ctrl.Services = NewServiceMap() + ctrl.Services.Set("ns", "local", &ServiceEndpoints{ + Service: lbService("192.0.2.10", annot), + Endpoint: epOnNode("10.0.0.1", "node-a"), + }) + ctrl.Services.Set("ns", "remote", &ServiceEndpoints{ + Service: lbService("192.0.2.11", annot), + Endpoint: epOnNode("10.0.0.2", "node-b"), + }) + // A service still waiting for its endpoint contributes to neither map. + ctrl.Services.Set("ns", "pending", &ServiceEndpoints{ + Service: lbService("192.0.2.12", annot), + Endpoint: nil, + }) + + if err := ctrl.cleanupRemovedServices(); err != nil { + t.Fatalf("cleanupRemovedServices: %v", err) + } + rec := ctrl.Proxy.(*recordingProxy) + + wantEgress := map[string]string{"192.0.2.10": "10.0.0.1", "192.0.2.11": "10.0.0.2"} + if len(rec.keepEgress) != len(wantEgress) { + t.Fatalf("keepEgress = %v, want %v", rec.keepEgress, wantEgress) + } + for svc, pod := range wantEgress { + if rec.keepEgress[svc] != pod { + t.Errorf("keepEgress[%s] = %q, want %q", svc, rec.keepEgress[svc], pod) + } + } + + wantIngress := map[string]string{"192.0.2.10": "10.0.0.1"} + if len(rec.keepIngress) != len(wantIngress) { + t.Fatalf("keepIngress = %v, want %v", rec.keepIngress, wantIngress) + } + if rec.keepIngress["192.0.2.10"] != "10.0.0.1" { + t.Errorf("keepIngress = %v, want %v", rec.keepIngress, wantIngress) + } +} diff --git a/pkg/proxy/dummy.go b/pkg/proxy/dummy.go index 3734c38..1c1aeea 100644 --- a/pkg/proxy/dummy.go +++ b/pkg/proxy/dummy.go @@ -13,18 +13,28 @@ func (d *DummyProxyProcessor) InitRules() error { return nil } -func (d *DummyProxyProcessor) EnsureRules(SvcIP, PodIP string) error { - fmt.Printf("EnsureRules called with SvcIP: %s, PodIP: %s\n", SvcIP, PodIP) +func (d *DummyProxyProcessor) EnsureEgressSNAT(SvcIP, PodIP string) error { + fmt.Printf("EnsureEgressSNAT called with SvcIP: %s, PodIP: %s\n", SvcIP, PodIP) return nil } -func (d *DummyProxyProcessor) DeleteRules(SvcIP, PodIP string) error { - fmt.Printf("DeleteRules called with SvcIP: %s, PodIP: %s\n", SvcIP, PodIP) +func (d *DummyProxyProcessor) DeleteEgressSNAT(SvcIP, PodIP string) error { + fmt.Printf("DeleteEgressSNAT called with SvcIP: %s, PodIP: %s\n", SvcIP, PodIP) return nil } -func (d *DummyProxyProcessor) CleanupRules(KeepMap map[string]string) error { - fmt.Println("CleanupRules called with KeepMap:", KeepMap) +func (d *DummyProxyProcessor) EnsureIngressDNAT(SvcIP, PodIP string) error { + fmt.Printf("EnsureIngressDNAT called with SvcIP: %s, PodIP: %s\n", SvcIP, PodIP) + return nil +} + +func (d *DummyProxyProcessor) DeleteIngressDNAT(SvcIP, PodIP string) error { + fmt.Printf("DeleteIngressDNAT called with SvcIP: %s, PodIP: %s\n", SvcIP, PodIP) + return nil +} + +func (d *DummyProxyProcessor) CleanupRules(keepEgress, keepIngress map[string]string) error { + fmt.Printf("CleanupRules called with keepEgress: %v, keepIngress: %v\n", keepEgress, keepIngress) return nil } diff --git a/pkg/proxy/interface.go b/pkg/proxy/interface.go index 4786120..f7c0e2b 100644 --- a/pkg/proxy/interface.go +++ b/pkg/proxy/interface.go @@ -4,9 +4,39 @@ import corev1 "k8s.io/api/core/v1" type ProxyProcessor interface { InitRules() error - EnsureRules(SvcIP, PodIP string) error - DeleteRules(SvcIP, PodIP string) error - CleanupRules(KeepMap map[string]string) error + + // EnsureEgressSNAT programs the pod_svc entry (pod IP → service IP) read + // by the egress_snat chain, so traffic leaving the backend is seen with + // the service IP as its source. + // + // Every node programs it, for every managed service, whether or not it + // hosts the backend. A reply from the backend to an intra-cluster client + // is handed straight to the client's node over the overlay: it never + // traverses the backend node's netfilter hooks, and the client's node is + // then the only place left where the source can still be rewritten. + EnsureEgressSNAT(SvcIP, PodIP string) error + + // DeleteEgressSNAT removes the pod_svc entry for the pair. No-op if absent. + DeleteEgressSNAT(SvcIP, PodIP string) error + + // EnsureIngressDNAT programs the svc_pod entry (service IP → pod IP) read + // by the ingress_dnat chain, so traffic addressed to the service IP is + // delivered to the backend. + // + // Only the node hosting the backend may program it. On any other node the + // rewrite would happen before the packet even leaves, and the hosting node + // would then record a conntrack tuple the reply can no longer match. + EnsureIngressDNAT(SvcIP, PodIP string) error + + // DeleteIngressDNAT removes the svc_pod entry for the pair. No-op if absent. + DeleteIngressDNAT(SvcIP, PodIP string) error + + // CleanupRules reconciles both maps against the desired state. Both + // arguments map service IP → pod IP: keepEgress covers every managed + // service, keepIngress only the backends hosted on this node. Anything + // else is removed, so state inherited from a build with different scoping + // is purged at startup. + CleanupRules(keepEgress, keepIngress map[string]string) error // EnsurePortFilter installs (or replaces) ingress port-filtering rules // for the given pod IP. Only TCP/UDP traffic destined to one of the diff --git a/pkg/proxy/nft.go b/pkg/proxy/nft.go index 1caa33f..5933745 100644 --- a/pkg/proxy/nft.go +++ b/pkg/proxy/nft.go @@ -387,146 +387,158 @@ func (p *NFTProxyProcessor) InitRules() error { return nil } -// EnsureRules ensures that a one-to-one mapping exists between svcIP and podIP. -// If a mapping already exists for svcIP with a different podIP, -// the old mapping is removed (from svc_pod, pod_svc, and from the raw pod set) -// before the new mapping is added. -func (p *NFTProxyProcessor) EnsureRules(svcIP, podIP string) error { - log.Info("Ensuring NAT mapping", "svcIP", svcIP, "podIP", podIP) - - parsedSvcIP := net.ParseIP(svcIP).To4() - if parsedSvcIP == nil { - return fmt.Errorf("invalid svcIP: %s", svcIP) - } - parsedPodIP := net.ParseIP(podIP).To4() - if parsedPodIP == nil { - return fmt.Errorf("invalid podIP: %s", podIP) - } +// EnsureEgressSNAT adds the pod_svc entry (podIP → svcIP) consulted by the +// egress_snat chain. If podIP is already mapped to another service, that stale +// entry is dropped first: a pod IP can only stand for one service IP. +// +// Programmed on every node, see ProxyProcessor.EnsureEgressSNAT. +func (p *NFTProxyProcessor) EnsureEgressSNAT(svcIP, podIP string) error { + log.Info("Ensuring egress SNAT", "svcIP", svcIP, "podIP", podIP) - // --- Remove conflicting mapping for svcIP in svc_pod map --- - // If svcIP already maps to a different pod, remove that mapping and - // delete the old pod from the raw pod set. - svcPodElems, err := p.conn.GetSetElements(p.svcPodMap) + parsedSvcIP, parsedPodIP, err := parsePair(svcIP, podIP) if err != nil { - log.Error(err, "Failed to get svc_pod map elements") - return fmt.Errorf("failed to get svc_pod map elements: %v", err) - } - for _, el := range svcPodElems { - if bytes.Equal(el.Key, parsedSvcIP) { - // Found an existing mapping for svcIP. - if !bytes.Equal(el.Val, parsedPodIP) { - oldPodIP := el.Val - log.Info("Updating mapping for svc", "svcIP", svcIP, "oldPodIP", net.IP(oldPodIP).String(), "newPodIP", podIP) - // Remove the old mapping from svc_pod. - if err := p.conn.SetDeleteElements(p.svcPodMap, []nftables.SetElement{{Key: parsedSvcIP, Val: oldPodIP}}); err != nil { - log.Error(err, "Failed to delete old svc_pod mapping", "svcIP", svcIP, "oldPodIP", net.IP(oldPodIP).String()) - return fmt.Errorf("failed to delete old svc_pod mapping: %v", err) - } - // Remove the corresponding mapping from pod_svc. - if err := p.conn.SetDeleteElements(p.podSvcMap, []nftables.SetElement{{Key: oldPodIP, Val: parsedSvcIP}}); err != nil { - log.Error(err, "Failed to delete corresponding pod_svc mapping", "oldPodIP", net.IP(oldPodIP).String(), "svcIP", svcIP) - return fmt.Errorf("failed to delete corresponding pod_svc mapping: %v", err) - } - } - break // svcIP mapping handled; exit loop. - } + return err } - // --- Remove conflicting mapping for podIP in pod_svc map --- - // If podIP already maps to a different svc, remove that mapping and delete the podIP - // from the raw pod set (since the old mapping is no longer desired). podSvcElems, err := p.conn.GetSetElements(p.podSvcMap) if err != nil { log.Error(err, "Failed to get pod_svc map elements") return fmt.Errorf("failed to get pod_svc map elements: %v", err) } for _, el := range podSvcElems { - if bytes.Equal(el.Key, parsedPodIP) { - // Found an existing mapping for podIP. - if !bytes.Equal(el.Val, parsedSvcIP) { - log.Info("Updating mapping for pod", "podIP", podIP, "oldSvcIP", net.IP(el.Val).String(), "newSvcIP", svcIP) - // Remove the old mapping from pod_svc. - if err := p.conn.SetDeleteElements(p.podSvcMap, []nftables.SetElement{{Key: parsedPodIP, Val: el.Val}}); err != nil { - log.Error(err, "Failed to delete old pod_svc mapping", "podIP", podIP, "oldSvcIP", net.IP(el.Val).String()) - return fmt.Errorf("failed to delete old pod_svc mapping: %v", err) - } - // Remove the corresponding mapping from svc_pod. - if err := p.conn.SetDeleteElements(p.svcPodMap, []nftables.SetElement{{Key: el.Val, Val: parsedPodIP}}); err != nil { - log.Error(err, "Failed to delete corresponding svc_pod mapping", "oldSvcIP", net.IP(el.Val).String(), "podIP", podIP) - return fmt.Errorf("failed to delete corresponding svc_pod mapping: %v", err) - } + if !bytes.Equal(el.Key, parsedPodIP) { + continue + } + if !bytes.Equal(el.Val, parsedSvcIP) { + log.Info("Updating egress SNAT for pod", "podIP", podIP, + "oldSvcIP", net.IP(el.Val).String(), "newSvcIP", svcIP) + if err := p.conn.SetDeleteElements(p.podSvcMap, []nftables.SetElement{{Key: parsedPodIP, Val: el.Val}}); err != nil { + log.Error(err, "Failed to delete stale pod_svc mapping", "podIP", podIP) + return fmt.Errorf("failed to delete stale pod_svc mapping: %v", err) } - break // podIP mapping handled; exit loop. } + break } - // --- Add the new mapping to both maps --- if err := p.conn.SetAddElements(p.podSvcMap, []nftables.SetElement{{Key: parsedPodIP, Val: parsedSvcIP}}); err != nil { log.Error(err, "Failed to add mapping to pod_svc", "podIP", podIP, "svcIP", svcIP) return fmt.Errorf("failed to add mapping to pod_svc: %v", err) } - if err := p.conn.SetAddElements(p.svcPodMap, []nftables.SetElement{{Key: parsedSvcIP, Val: parsedPodIP}}); err != nil { - log.Error(err, "Failed to add mapping to svc_pod", "svcIP", svcIP, "podIP", podIP) - return fmt.Errorf("failed to add mapping to svc_pod: %v", err) - } - log.Info("Added mapping", "svcIP", svcIP, "podIP", podIP) - - // Commit all changes. if err := p.conn.Flush(); err != nil { - log.Error(err, "Failed to commit EnsureNAT changes") - return fmt.Errorf("failed to commit EnsureNAT changes: %v", err) + log.Error(err, "Failed to commit egress SNAT changes") + return fmt.Errorf("failed to commit egress SNAT changes: %v", err) } - log.Info("NAT mapping ensured successfully", "svcIP", svcIP, "podIP", podIP) + log.Info("Egress SNAT ensured successfully", "svcIP", svcIP, "podIP", podIP) return nil } -// DeleteRules removes the mapping for the given svcIP and podIP from both maps -// and commits the removal from NAT translation maps. -func (p *NFTProxyProcessor) DeleteRules(svcIP, podIP string) error { - log.Info("Deleting NAT mapping", "svcIP", svcIP, "podIP", podIP) +// DeleteEgressSNAT removes the pod_svc entry for the pair. An entry that is +// already gone is not an error. +func (p *NFTProxyProcessor) DeleteEgressSNAT(svcIP, podIP string) error { + log.Info("Deleting egress SNAT", "svcIP", svcIP, "podIP", podIP) - // Parse svcIP and podIP into IPv4 byte slices. - parsedSvcIP := net.ParseIP(svcIP).To4() - if parsedSvcIP == nil { - return fmt.Errorf("invalid svcIP: %s", svcIP) - } - parsedPodIP := net.ParseIP(podIP).To4() - if parsedPodIP == nil { - return fmt.Errorf("invalid podIP: %s", podIP) + parsedSvcIP, parsedPodIP, err := parsePair(svcIP, podIP) + if err != nil { + return err } - // Delete mapping from the "pod_svc" map. if err := p.conn.SetDeleteElements(p.podSvcMap, []nftables.SetElement{ {Key: parsedPodIP, Val: parsedSvcIP}, }); err != nil { log.Error(err, "Failed to delete mapping from pod_svc", "podIP", podIP, "svcIP", svcIP) return fmt.Errorf("failed to delete mapping from pod_svc: %v", err) } + if err := p.flushTolerateENOENT("DeleteEgressSNAT"); err != nil { + log.Error(err, "Failed to commit egress SNAT deletion") + return fmt.Errorf("failed to commit egress SNAT deletion: %v", err) + } + log.Info("Egress SNAT deleted successfully", "svcIP", svcIP, "podIP", podIP) + return nil +} + +// EnsureIngressDNAT adds the svc_pod entry (svcIP → podIP) consulted by the +// ingress_dnat chain. If svcIP is already mapped to another pod — a migrated +// VM, a replaced backend — that stale entry is dropped first. +// +// Programmed only by the node hosting the backend, see +// ProxyProcessor.EnsureIngressDNAT. +func (p *NFTProxyProcessor) EnsureIngressDNAT(svcIP, podIP string) error { + log.Info("Ensuring ingress DNAT", "svcIP", svcIP, "podIP", podIP) + + parsedSvcIP, parsedPodIP, err := parsePair(svcIP, podIP) + if err != nil { + return err + } + + svcPodElems, err := p.conn.GetSetElements(p.svcPodMap) + if err != nil { + log.Error(err, "Failed to get svc_pod map elements") + return fmt.Errorf("failed to get svc_pod map elements: %v", err) + } + for _, el := range svcPodElems { + if !bytes.Equal(el.Key, parsedSvcIP) { + continue + } + if !bytes.Equal(el.Val, parsedPodIP) { + log.Info("Updating ingress DNAT for svc", "svcIP", svcIP, + "oldPodIP", net.IP(el.Val).String(), "newPodIP", podIP) + if err := p.conn.SetDeleteElements(p.svcPodMap, []nftables.SetElement{{Key: parsedSvcIP, Val: el.Val}}); err != nil { + log.Error(err, "Failed to delete stale svc_pod mapping", "svcIP", svcIP) + return fmt.Errorf("failed to delete stale svc_pod mapping: %v", err) + } + } + break + } + + if err := p.conn.SetAddElements(p.svcPodMap, []nftables.SetElement{{Key: parsedSvcIP, Val: parsedPodIP}}); err != nil { + log.Error(err, "Failed to add mapping to svc_pod", "svcIP", svcIP, "podIP", podIP) + return fmt.Errorf("failed to add mapping to svc_pod: %v", err) + } + if err := p.conn.Flush(); err != nil { + log.Error(err, "Failed to commit ingress DNAT changes") + return fmt.Errorf("failed to commit ingress DNAT changes: %v", err) + } + log.Info("Ingress DNAT ensured successfully", "svcIP", svcIP, "podIP", podIP) + return nil +} + +// DeleteIngressDNAT removes the svc_pod entry for the pair. An entry that is +// already gone is not an error. +func (p *NFTProxyProcessor) DeleteIngressDNAT(svcIP, podIP string) error { + log.Info("Deleting ingress DNAT", "svcIP", svcIP, "podIP", podIP) + + parsedSvcIP, parsedPodIP, err := parsePair(svcIP, podIP) + if err != nil { + return err + } - // Delete mapping from the "svc_pod" map. if err := p.conn.SetDeleteElements(p.svcPodMap, []nftables.SetElement{ {Key: parsedSvcIP, Val: parsedPodIP}, }); err != nil { log.Error(err, "Failed to delete mapping from svc_pod", "svcIP", svcIP, "podIP", podIP) return fmt.Errorf("failed to delete mapping from svc_pod: %v", err) } - - // Commit all changes. - if err := p.conn.Flush(); err != nil { - // Check if the error is ENOENT (no such file or directory) and ignore it. - // This may happen if the elements or even the table were already removed. - if errors.Is(err, unix.ENOENT) { - log.Info("Ignoring ENOENT error during flush in DeleteRules", "error", err) - } else { - log.Error(err, "Failed to commit DeleteNAT changes") - return fmt.Errorf("failed to commit DeleteNAT changes: %v", err) - } + if err := p.flushTolerateENOENT("DeleteIngressDNAT"); err != nil { + log.Error(err, "Failed to commit ingress DNAT deletion") + return fmt.Errorf("failed to commit ingress DNAT deletion: %v", err) } - - log.Info("NAT mapping and raw set elements deleted successfully", "svcIP", svcIP, "podIP", podIP) + log.Info("Ingress DNAT deleted successfully", "svcIP", svcIP, "podIP", podIP) return nil } +// parsePair validates a (svcIP, podIP) pair and returns both as IPv4 bytes. +func parsePair(svcIP, podIP string) (net.IP, net.IP, error) { + parsedSvcIP := net.ParseIP(svcIP).To4() + if parsedSvcIP == nil { + return nil, nil, fmt.Errorf("invalid svcIP: %s", svcIP) + } + parsedPodIP := net.ParseIP(podIP).To4() + if parsedPodIP == nil { + return nil, nil, fmt.Errorf("invalid podIP: %s", podIP) + } + return parsedSvcIP, parsedPodIP, nil +} + // flushTolerateENOENT commits the pending batch and treats ENOENT as success. // // Deleting a set element that is already gone reports ENOENT, which fails the @@ -545,103 +557,77 @@ func (p *NFTProxyProcessor) flushTolerateENOENT(op string) error { return err } -// CleanupRules receives a keepMap (keys: svcIP, values: podIP) representing the desired state. -// It recovers from an inconsistent state by: -// 1. Removing any mappings in the pod_svc and svc_pod maps that do not match keepMap. -// 2. Adding any missing mappings from keepMap into both maps. -// 3. Cleaning up the raw sets (pod and svc) so that only the desired IPs remain. -func (p *NFTProxyProcessor) CleanupRules(keepMap map[string]string) error { - log.Info("Starting CleanupRules", "keepMap", keepMap) +// CleanupRules reconciles both NAT maps against the desired state. +// +// keepEgress and keepIngress both map service IP → pod IP. They differ in +// scope, which is the whole point: pod_svc must carry every managed service so +// a reply reaching this node over the overlay still gets its source rewritten, +// while svc_pod must carry only the backends hosted here so no other node +// translates a destination before the packet has left. +// +// Anything present in a map but absent from its keep set is removed, which is +// what purges state inherited from a build that scoped the two maps alike. +func (p *NFTProxyProcessor) CleanupRules(keepEgress, keepIngress map[string]string) error { + log.Info("Starting CleanupRules", "keepEgress", keepEgress, "keepIngress", keepIngress) - // --- Step 1: Clean up mapping sets --- + // --- Step 1: collect what has to change in both maps --- - // Retrieve current mappings from the pod_svc map. - // Note: pod_svc maps pod IP → svc IP. - podSvcElems, err := p.conn.GetSetElements(p.podSvcMap) + // pod_svc is keyed by pod IP, so the desired state is keepEgress inverted. + desiredPodSvc := make(map[string]string, len(keepEgress)) // pod → svc + for svc, pod := range keepEgress { + desiredPodSvc[pod] = svc + } + + podSvcDel, podSvcAdd, err := p.diffMap(p.podSvcMap, desiredPodSvc) if err != nil { - log.Error(err, "Failed to get pod_svc elements") - return fmt.Errorf("failed to get pod_svc elements: %v", err) + return fmt.Errorf("failed to diff pod_svc: %v", err) + } + svcPodDel, svcPodAdd, err := p.diffMap(p.svcPodMap, keepIngress) + if err != nil { + return fmt.Errorf("failed to diff svc_pod: %v", err) } - // Build a current mapping in svc->pod direction (for easy comparison with keepMap) - currentMapping := make(map[string]string) // key: svc, value: pod - for _, el := range podSvcElems { - pod := net.IP(el.Key).String() - svc := net.IP(el.Val).String() - currentMapping[svc] = pod - } - - // Prepare slices for elements to delete from both maps. - var toDeletePodSvc []nftables.SetElement - var toDeleteSvcPod []nftables.SetElement - - // For each mapping found in the current configuration, if it does not match the desired state, mark it for deletion. - for svc, pod := range currentMapping { - if expectedPod, ok := keepMap[svc]; !ok || expectedPod != pod { - log.Info("Marking inconsistent mapping for deletion", "svcIP", svc, "podIP", pod) - // Prepare deletion elements. - // pod_svc: key = pod, val = svc. - toDeletePodSvc = append(toDeletePodSvc, nftables.SetElement{ - Key: net.ParseIP(pod).To4(), - Val: net.ParseIP(svc).To4(), - }) - // svc_pod: key = svc, val = pod. - toDeleteSvcPod = append(toDeleteSvcPod, nftables.SetElement{ - Key: net.ParseIP(svc).To4(), - Val: net.ParseIP(pod).To4(), - }) - } - } - - // Delete any inconsistent mappings. - if len(toDeletePodSvc) > 0 { - if err := p.conn.SetDeleteElements(p.podSvcMap, toDeletePodSvc); err != nil { - log.Error(err, "Failed to delete inconsistent mappings from pod_svc") - return fmt.Errorf("failed to delete inconsistent mappings from pod_svc: %v", err) - } - if err := p.conn.SetDeleteElements(p.svcPodMap, toDeleteSvcPod); err != nil { - log.Error(err, "Failed to delete inconsistent mappings from svc_pod") - return fmt.Errorf("failed to delete inconsistent mappings from svc_pod: %v", err) - } - // Commit the deletions before queueing the additions below: an element - // that is already gone fails the flush, and a shared batch would report - // that as a cleanup failure, which aborts the controller at startup. + // --- Step 2: commit the deletions on their own --- + // + // An element that is already gone fails the flush, and a shared batch + // would report that as a cleanup failure, aborting the controller at + // startup. + if len(podSvcDel) > 0 || len(svcPodDel) > 0 { + if len(podSvcDel) > 0 { + if err := p.conn.SetDeleteElements(p.podSvcMap, podSvcDel); err != nil { + log.Error(err, "Failed to delete stale mappings from pod_svc") + return fmt.Errorf("failed to delete stale mappings from pod_svc: %v", err) + } + } + if len(svcPodDel) > 0 { + if err := p.conn.SetDeleteElements(p.svcPodMap, svcPodDel); err != nil { + log.Error(err, "Failed to delete stale mappings from svc_pod") + return fmt.Errorf("failed to delete stale mappings from svc_pod: %v", err) + } + } if err := p.flushTolerateENOENT("CleanupRules deletions"); err != nil { log.Error(err, "Failed to commit cleanup deletions") return fmt.Errorf("failed to commit cleanup deletions: %v", err) } - log.Info("Inconsistent mappings removed from both maps") + log.Info("Stale mappings removed", "podSvc", len(podSvcDel), "svcPod", len(svcPodDel)) } else { - log.Info("No inconsistent mappings found in maps") + log.Info("No stale mappings found in maps") } - // --- Step 2: Add missing mappings from keepMap --- - - // For every desired mapping in keepMap, ensure it exists in both maps. - for svc, pod := range keepMap { - // Check if the current mapping for svc exists and matches. - if existingPod, ok := currentMapping[svc]; !ok || existingPod != pod { - parsedSvcIP := net.ParseIP(svc).To4() - parsedPodIP := net.ParseIP(pod).To4() - if parsedSvcIP == nil || parsedPodIP == nil { - log.Error(nil, "Invalid IP in keepMap", "svcIP", svc, "podIP", pod) - continue - } - // Add mapping to pod_svc (pod → svc) - if err := p.conn.SetAddElements(p.podSvcMap, []nftables.SetElement{{Key: parsedPodIP, Val: parsedSvcIP}}); err != nil { - log.Error(err, "Failed to add missing mapping to pod_svc", "podIP", pod, "svcIP", svc) - return fmt.Errorf("failed to add missing mapping to pod_svc: %v", err) - } - // Add mapping to svc_pod (svc → pod) - if err := p.conn.SetAddElements(p.svcPodMap, []nftables.SetElement{{Key: parsedSvcIP, Val: parsedPodIP}}); err != nil { - log.Error(err, "Failed to add missing mapping to svc_pod", "svcIP", svc, "podIP", pod) - return fmt.Errorf("failed to add missing mapping to svc_pod: %v", err) - } - log.Info("Added missing mapping", "svcIP", svc, "podIP", pod) + // --- Step 3: add whatever is missing --- + if len(podSvcAdd) > 0 { + if err := p.conn.SetAddElements(p.podSvcMap, podSvcAdd); err != nil { + log.Error(err, "Failed to add missing mappings to pod_svc") + return fmt.Errorf("failed to add missing mappings to pod_svc: %v", err) + } + } + if len(svcPodAdd) > 0 { + if err := p.conn.SetAddElements(p.svcPodMap, svcPodAdd); err != nil { + log.Error(err, "Failed to add missing mappings to svc_pod") + return fmt.Errorf("failed to add missing mappings to svc_pod: %v", err) } } - // --- Final commit --- // Startup cleanup must not be fatal: aborting here takes the whole // DaemonSet pod down and leaves the node's datapath half-programmed, while // the reconcile loop would have converged on the next event anyway. @@ -649,10 +635,48 @@ func (p *NFTProxyProcessor) CleanupRules(keepMap map[string]string) error { log.Error(err, "Failed to commit cleanup changes") return fmt.Errorf("failed to commit cleanup changes: %v", err) } - log.Info("CleanupRules completed successfully") + log.Info("CleanupRules completed successfully", + "podSvcAdded", len(podSvcAdd), "svcPodAdded", len(svcPodAdd)) return nil } +// diffMap compares one nft map against its desired state and returns the +// elements to delete and to add. Both the map and desired are read in the +// map's own key → value direction. +func (p *NFTProxyProcessor) diffMap(m *nftables.Set, desired map[string]string) (del, add []nftables.SetElement, err error) { + elems, err := p.conn.GetSetElements(m) + if err != nil { + log.Error(err, "Failed to get map elements", "map", m.Name) + return nil, nil, fmt.Errorf("failed to get %s elements: %v", m.Name, err) + } + + current := make(map[string]string, len(elems)) + for _, el := range elems { + key := net.IP(el.Key).String() + val := net.IP(el.Val).String() + current[key] = val + if want, ok := desired[key]; !ok || want != val { + log.Info("Marking stale mapping for deletion", "map", m.Name, "key", key, "value", val) + del = append(del, nftables.SetElement{Key: el.Key, Val: el.Val}) + } + } + + for key, val := range desired { + if cur, ok := current[key]; ok && cur == val { + continue + } + parsedKey := net.ParseIP(key).To4() + parsedVal := net.ParseIP(val).To4() + if parsedKey == nil || parsedVal == nil { + log.Error(nil, "Invalid IP in desired state", "map", m.Name, "key", key, "value", val) + continue + } + log.Info("Adding missing mapping", "map", m.Name, "key", key, "value", val) + add = append(add, nftables.SetElement{Key: parsedKey, Val: parsedVal}) + } + return del, add, nil +} + // EnsurePortFilter installs ingress port filtering rules for the given // (svcIP, podIP) pair. The actual nft set keys are pod IPs, because the // port_filter chain runs at priority filter (0), after ingress_dnat has From 1110b24d0e06b5996c9668e64023662d948fc3ee Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 15 Sep 2026 13:17:23 +0200 Subject: [PATCH 04/42] proxy: retry cleanup deletions per element on ENOENT A flush is a single nftables transaction, so one element that is already gone aborts every other deletion queued with it. The startup purges tolerated that ENOENT on the whole batch, which reported success while removing nothing. Observed on a three-node cluster while rolling the image back: the node had seven stale pod_svc entries, the purge queued all seven plus the mirrored svc_pod deletions, svc_pod was empty, and the resulting ENOENT cancelled the pod_svc deletions too. The log said the cleanup completed successfully and the map still held every entry. Retry element by element when the batch reports ENOENT, so only the elements that really are gone are skipped. Applies to all three startup purges: the NAT maps, the port filters and the ICMP allowlist. Tolerating ENOENT on a single-element deletion stays as it was, since there is nothing else in the batch to lose. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5 (1M context) --- pkg/proxy/nft.go | 98 +++++++++++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/pkg/proxy/nft.go b/pkg/proxy/nft.go index 5933745..ec87e89 100644 --- a/pkg/proxy/nft.go +++ b/pkg/proxy/nft.go @@ -557,6 +557,51 @@ func (p *NFTProxyProcessor) flushTolerateENOENT(op string) error { return err } +// deleteElementsTolerant removes elements from a set, falling back to one +// commit per element when a batched commit reports that one of them is gone. +// +// A flush is a single nftables transaction: one missing element aborts every +// other deletion queued with it. Tolerating that ENOENT on the batch would +// report success while removing nothing, leaving exactly the stale state the +// purge exists to clear — which is how a node can keep entries belonging to a +// scope it no longer programs. Retrying per element skips only the ones that +// really are gone. +func (p *NFTProxyProcessor) deleteElementsTolerant(m *nftables.Set, elems []nftables.SetElement, op string) error { + if len(elems) == 0 { + return nil + } + + if err := p.conn.SetDeleteElements(m, elems); err != nil { + return fmt.Errorf("failed to queue deletions for %s: %v", m.Name, err) + } + err := p.conn.Flush() + if err == nil { + return nil + } + if !errors.Is(err, unix.ENOENT) { + return fmt.Errorf("failed to commit deletions for %s: %v", m.Name, err) + } + + log.Info("Batched deletion hit ENOENT, retrying element by element", + "op", op, "set", m.Name, "elements", len(elems)) + removed := 0 + for _, el := range elems { + if err := p.conn.SetDeleteElements(m, []nftables.SetElement{el}); err != nil { + return fmt.Errorf("failed to queue deletion for %s: %v", m.Name, err) + } + if err := p.conn.Flush(); err != nil { + if errors.Is(err, unix.ENOENT) { + continue // already gone, nothing to do + } + return fmt.Errorf("failed to commit deletion for %s: %v", m.Name, err) + } + removed++ + } + log.Info("Element-by-element deletion completed", + "op", op, "set", m.Name, "removed", removed, "alreadyGone", len(elems)-removed) + return nil +} + // CleanupRules reconciles both NAT maps against the desired state. // // keepEgress and keepIngress both map service IP → pod IP. They differ in @@ -593,21 +638,13 @@ func (p *NFTProxyProcessor) CleanupRules(keepEgress, keepIngress map[string]stri // would report that as a cleanup failure, aborting the controller at // startup. if len(podSvcDel) > 0 || len(svcPodDel) > 0 { - if len(podSvcDel) > 0 { - if err := p.conn.SetDeleteElements(p.podSvcMap, podSvcDel); err != nil { - log.Error(err, "Failed to delete stale mappings from pod_svc") - return fmt.Errorf("failed to delete stale mappings from pod_svc: %v", err) - } + if err := p.deleteElementsTolerant(p.podSvcMap, podSvcDel, "CleanupRules"); err != nil { + log.Error(err, "Failed to delete stale mappings from pod_svc") + return err } - if len(svcPodDel) > 0 { - if err := p.conn.SetDeleteElements(p.svcPodMap, svcPodDel); err != nil { - log.Error(err, "Failed to delete stale mappings from svc_pod") - return fmt.Errorf("failed to delete stale mappings from svc_pod: %v", err) - } - } - if err := p.flushTolerateENOENT("CleanupRules deletions"); err != nil { - log.Error(err, "Failed to commit cleanup deletions") - return fmt.Errorf("failed to commit cleanup deletions: %v", err) + if err := p.deleteElementsTolerant(p.svcPodMap, svcPodDel, "CleanupRules"); err != nil { + log.Error(err, "Failed to delete stale mappings from svc_pod") + return err } log.Info("Stale mappings removed", "podSvc", len(podSvcDel), "svcPod", len(svcPodDel)) } else { @@ -845,23 +882,14 @@ func (p *NFTProxyProcessor) CleanupPortFilters(keep map[string]PortFilterEntry) } } - // 4. Batch ops. - if len(delPods) > 0 { - if err := p.conn.SetDeleteElements(p.filteredPods, delPods); err != nil { - return fmt.Errorf("failed to delete stale filtered_pods: %v", err) - } - } - if len(delPorts) > 0 { - if err := p.conn.SetDeleteElements(p.allowedPorts, delPorts); err != nil { - return fmt.Errorf("failed to delete stale allowed_ports: %v", err) - } + // 4. Batch ops. Deletions are committed first and on their own, so an + // element that is already gone cannot fail the batch carrying the + // additions below. + if err := p.deleteElementsTolerant(p.filteredPods, delPods, "CleanupPortFilters"); err != nil { + return err } - // Commit deletions separately so an element that is already gone cannot - // fail the batch carrying the additions below. - if len(delPods) > 0 || len(delPorts) > 0 { - if err := p.flushTolerateENOENT("CleanupPortFilters deletions"); err != nil { - return fmt.Errorf("failed to flush CleanupPortFilters deletions: %v", err) - } + if err := p.deleteElementsTolerant(p.allowedPorts, delPorts, "CleanupPortFilters"); err != nil { + return err } if len(addPods) > 0 { if err := p.conn.SetAddElements(p.filteredPods, addPods); err != nil { @@ -1003,14 +1031,8 @@ func (p *NFTProxyProcessor) CleanupICMPAllow(keep map[string]string) error { } } - if len(delPods) > 0 { - if err := p.conn.SetDeleteElements(p.icmpAllowedPods, delPods); err != nil { - return fmt.Errorf("failed to delete stale icmp_allowed_pods: %v", err) - } - // Commit deletions separately: see flushTolerateENOENT. - if err := p.flushTolerateENOENT("CleanupICMPAllow deletions"); err != nil { - return fmt.Errorf("failed to flush CleanupICMPAllow deletions: %v", err) - } + if err := p.deleteElementsTolerant(p.icmpAllowedPods, delPods, "CleanupICMPAllow"); err != nil { + return err } if len(addPods) > 0 { if err := p.conn.SetAddElements(p.icmpAllowedPods, addPods); err != nil { From 0cdd98446fe2ba480c310644066a21283552575e Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 15 Sep 2026 14:01:14 +0200 Subject: [PATCH 05/42] proxy: serialize datapath writes and isolate conflict removals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four nodes lost one service each during a rolling restart, with the same commit error: "conn.Receive: netlink receive: no such file or directory". One of them, a public IP whose ingress DNAT never landed, stayed unreachable for nine minutes until its pod was restarted. nftables.Conn guards its message list but not the interval between queueing messages and committing them, and this connection is shared by the service informer, the endpoint informer and the startup reconciliation. One caller's flush therefore carries another's half-queued messages. A batch is a single transaction, so a stale deletion in it aborts the other caller's addition — and nothing retries, because the controller only reacts to events. The service stays unprogrammed until the next one. Take a mutex across every queue-then-flush sequence, so a batch only ever holds one caller's messages. EnsurePortFilter reaches DeletePortFilter through an unlocked inner function, since sync.Mutex is not reentrant. Also commit each conflict removal on its own, ahead of the addition that replaces it. Serialization alone leaves the ordinary case where the element read a moment earlier is already gone: the removal fails, and with it the addition queued behind it. Splitting them keeps the addition independent of an ENOENT that no longer matters. This is what made EnsurePortFilter dangerous — its rebuild deletes the pod's allowed_ports before re-adding them, so losing the additions leaves the pod in filtered_pods with no open port at all. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5 (1M context) --- pkg/proxy/nft.go | 78 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/pkg/proxy/nft.go b/pkg/proxy/nft.go index ec87e89..7cb296b 100644 --- a/pkg/proxy/nft.go +++ b/pkg/proxy/nft.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "sync" "github.com/google/nftables" "github.com/google/nftables/binaryutil" @@ -18,6 +19,18 @@ var log = ctrl.Log.WithName("nft-proxy-processor") // NFTProxyProcessor implements a NATProcessor using nftables. type NFTProxyProcessor struct { + // mu serializes every queue-then-flush sequence below. + // + // nftables.Conn guards its own message list, but not the interval between + // queueing messages and committing them. Two informer goroutines — plus + // the startup reconciliation — share this connection, so without this + // lock one flush carries the other caller's half-queued messages. A batch + // is a single transaction: one stale deletion in it aborts the other + // caller's addition, which is then never retried because the controller + // only reacts to events. The service stays unprogrammed until the next + // one, which is how a public IP can go dark for minutes after a restart. + mu sync.Mutex + conn *nftables.Conn // Table "cozy_proxy" will contain all objects. @@ -37,6 +50,9 @@ type NFTProxyProcessor struct { // InitRules initializes the nftables configuration in a single table "cozy_proxy". // It flushes the entire ruleset, then re-creates the table with the desired sets, maps, and chains. func (p *NFTProxyProcessor) InitRules() error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Initializing nftables NAT configuration") // Create a new connection if needed. @@ -393,6 +409,9 @@ func (p *NFTProxyProcessor) InitRules() error { // // Programmed on every node, see ProxyProcessor.EnsureEgressSNAT. func (p *NFTProxyProcessor) EnsureEgressSNAT(svcIP, podIP string) error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Ensuring egress SNAT", "svcIP", svcIP, "podIP", podIP) parsedSvcIP, parsedPodIP, err := parsePair(svcIP, podIP) @@ -416,6 +435,13 @@ func (p *NFTProxyProcessor) EnsureEgressSNAT(svcIP, podIP string) error { log.Error(err, "Failed to delete stale pod_svc mapping", "podIP", podIP) return fmt.Errorf("failed to delete stale pod_svc mapping: %v", err) } + // Commit the removal on its own. Sharing the batch with the + // addition below would let an element that is already gone abort + // the addition too, and nothing retries it. + if err := p.flushTolerateENOENT("EnsureEgressSNAT conflict removal"); err != nil { + log.Error(err, "Failed to commit stale pod_svc removal", "podIP", podIP) + return fmt.Errorf("failed to commit stale pod_svc removal: %v", err) + } } break } @@ -435,6 +461,9 @@ func (p *NFTProxyProcessor) EnsureEgressSNAT(svcIP, podIP string) error { // DeleteEgressSNAT removes the pod_svc entry for the pair. An entry that is // already gone is not an error. func (p *NFTProxyProcessor) DeleteEgressSNAT(svcIP, podIP string) error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Deleting egress SNAT", "svcIP", svcIP, "podIP", podIP) parsedSvcIP, parsedPodIP, err := parsePair(svcIP, podIP) @@ -463,6 +492,9 @@ func (p *NFTProxyProcessor) DeleteEgressSNAT(svcIP, podIP string) error { // Programmed only by the node hosting the backend, see // ProxyProcessor.EnsureIngressDNAT. func (p *NFTProxyProcessor) EnsureIngressDNAT(svcIP, podIP string) error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Ensuring ingress DNAT", "svcIP", svcIP, "podIP", podIP) parsedSvcIP, parsedPodIP, err := parsePair(svcIP, podIP) @@ -486,6 +518,11 @@ func (p *NFTProxyProcessor) EnsureIngressDNAT(svcIP, podIP string) error { log.Error(err, "Failed to delete stale svc_pod mapping", "svcIP", svcIP) return fmt.Errorf("failed to delete stale svc_pod mapping: %v", err) } + // Commit the removal on its own, see EnsureEgressSNAT. + if err := p.flushTolerateENOENT("EnsureIngressDNAT conflict removal"); err != nil { + log.Error(err, "Failed to commit stale svc_pod removal", "svcIP", svcIP) + return fmt.Errorf("failed to commit stale svc_pod removal: %v", err) + } } break } @@ -505,6 +542,9 @@ func (p *NFTProxyProcessor) EnsureIngressDNAT(svcIP, podIP string) error { // DeleteIngressDNAT removes the svc_pod entry for the pair. An entry that is // already gone is not an error. func (p *NFTProxyProcessor) DeleteIngressDNAT(svcIP, podIP string) error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Deleting ingress DNAT", "svcIP", svcIP, "podIP", podIP) parsedSvcIP, parsedPodIP, err := parsePair(svcIP, podIP) @@ -613,6 +653,9 @@ func (p *NFTProxyProcessor) deleteElementsTolerant(m *nftables.Set, elems []nfta // Anything present in a map but absent from its keep set is removed, which is // what purges state inherited from a build that scoped the two maps alike. func (p *NFTProxyProcessor) CleanupRules(keepEgress, keepIngress map[string]string) error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Starting CleanupRules", "keepEgress", keepEgress, "keepIngress", keepIngress) // --- Step 1: collect what has to change in both maps --- @@ -721,13 +764,16 @@ func (p *NFTProxyProcessor) diffMap(m *nftables.Set, desired map[string]string) // permitted on the post-DNAT pod IP; all other traffic destined to that // pod IP is dropped. Idempotent. func (p *NFTProxyProcessor) EnsurePortFilter(svcIP, podIP string, ports []corev1.ServicePort) error { + p.mu.Lock() + defer p.mu.Unlock() + // Empty ports list is documented as equivalent to DeletePortFilter: // the caller wants to disable filtering for this pod entirely. Without // this short-circuit we would add the pod to filtered_pods with no // matching allowed_ports entries, which would drop every ingress packet // to the pod IP — the opposite of "no filter". if len(ports) == 0 { - return p.DeletePortFilter(svcIP, podIP) + return p.deletePortFilterLocked(svcIP, podIP) } log.Info("Ensuring port filter", "svcIP", svcIP, "podIP", podIP, "portCount", len(ports)) @@ -737,10 +783,16 @@ func (p *NFTProxyProcessor) EnsurePortFilter(svcIP, podIP string, ports []corev1 } // 1. Remove any pre-existing entries in allowed_ports for podIP so we can - // rebuild the tuple set cleanly (idempotent). + // rebuild the tuple set cleanly (idempotent). Committed on its own: an + // element that is already gone would otherwise abort the additions below + // in the same transaction, leaving the pod in filtered_pods with no + // allowed port — every packet to it dropped. if err := p.removeAllowedPortsForPod(parsedPodIP); err != nil { return err } + if err := p.flushTolerateENOENT("EnsurePortFilter port rebuild"); err != nil { + return fmt.Errorf("failed to commit allowed_ports rebuild for pod %s: %v", podIP, err) + } // 2. Add podIP to filtered_pods (idempotent — Add ignores duplicates). if err := p.conn.SetAddElements(p.filteredPods, []nftables.SetElement{{Key: parsedPodIP}}); err != nil { @@ -780,6 +832,16 @@ func (p *NFTProxyProcessor) EnsurePortFilter(svcIP, podIP string, ports []corev1 // port entries for it. svcIP is used only for logging context. Tolerates // ENOENT for clean idempotency. func (p *NFTProxyProcessor) DeletePortFilter(svcIP, podIP string) error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.deletePortFilterLocked(svcIP, podIP) +} + +// deletePortFilterLocked is DeletePortFilter without the lock, for callers +// that already hold it. sync.Mutex is not reentrant, so EnsurePortFilter must +// come through here rather than calling the exported method. +func (p *NFTProxyProcessor) deletePortFilterLocked(svcIP, podIP string) error { log.Info("Deleting port filter", "svcIP", svcIP, "podIP", podIP) parsedPodIP := net.ParseIP(podIP).To4() if parsedPodIP == nil { @@ -811,6 +873,9 @@ func (p *NFTProxyProcessor) DeletePortFilter(svcIP, podIP string) error { // batches SetAddElements / SetDeleteElements per set, and Flushes exactly // once. func (p *NFTProxyProcessor) CleanupPortFilters(keep map[string]PortFilterEntry) error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Starting CleanupPortFilters", "keepCount", len(keep)) // 1. Build desired state in memory. @@ -952,6 +1017,9 @@ func (p *NFTProxyProcessor) removeAllowedPortsForPod(parsedPodIP net.IP) error { // traffic to that pod IP bypasses the port_filter drop rule. svcIP is used // only for logging context. Idempotent. func (p *NFTProxyProcessor) EnsureICMPAllow(svcIP, podIP string) error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Ensuring ICMP allow", "svcIP", svcIP, "podIP", podIP) parsedPodIP := net.ParseIP(podIP).To4() if parsedPodIP == nil { @@ -970,6 +1038,9 @@ func (p *NFTProxyProcessor) EnsureICMPAllow(svcIP, podIP string) error { // DeleteICMPAllow removes podIP from icmp_allowed_pods. Tolerates ENOENT for // clean idempotency. svcIP is used only for logging context. func (p *NFTProxyProcessor) DeleteICMPAllow(svcIP, podIP string) error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Deleting ICMP allow", "svcIP", svcIP, "podIP", podIP) parsedPodIP := net.ParseIP(podIP).To4() if parsedPodIP == nil { @@ -996,6 +1067,9 @@ func (p *NFTProxyProcessor) DeleteICMPAllow(svcIP, podIP string) error { // must remain in the set. Single-pass diff with one Flush, mirroring // CleanupPortFilters. func (p *NFTProxyProcessor) CleanupICMPAllow(keep map[string]string) error { + p.mu.Lock() + defer p.mu.Unlock() + log.Info("Starting CleanupICMPAllow", "keepCount", len(keep)) desired := make(map[string]bool, len(keep)) From 76f25488325f65dc8f6d062801edacd51449d02d Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 15 Sep 2026 14:29:08 +0200 Subject: [PATCH 06/42] proxy: retry failed datapath writes, commit additions strictly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on two gaps that the production incident had already demonstrated, plus one it had not. A refused commit was logged and forgotten. The informers only deliver events, so a service whose programming failed stayed unprogrammed until the next one — which is how a public IP stayed dark for nine minutes. Queue the service instead and re-attempt every 30s until a pass is clean. Every datapath call is idempotent, so a re-attempt on something already correct costs nothing. reconcilePortFilter now returns an error so the caller can see it; a filtered pod with no allowed_ports drops every packet, which is the failure worth retrying hardest. A failed startup reconciliation was logged too, and would otherwise wait for the next event or the 12-hour informer resync. It goes through the same queue and stays non-fatal. Additions are now committed with a strict flush. flushTolerateENOENT was used after the addition batches in all three startup purges, where an ENOENT means the table or the set is missing, not that an element was already gone — so it reported a successful reconciliation with nothing installed. Only deletions may tolerate it. Also recorded why errors.Is(err, unix.ENOENT) does hold on the path the kernel actually takes: the errno arrives as a netlink.OpError wrapping a syscall.Errno, which unwraps. github.com/google/nftables v0.3.0 has one branch in receiveAckAware that formats a trailing error ack instead of wrapping it; v0.3.0 is the latest release, and an ENOENT arriving that way is surfaced as a plain error rather than silently swallowed, which is the safe direction. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5 (1M context) --- pkg/controllers/services_controller.go | 160 +++++++++++++++++++- pkg/controllers/services_controller_test.go | 142 +++++++++++++++++ pkg/proxy/nft.go | 21 ++- 3 files changed, 315 insertions(+), 8 deletions(-) diff --git a/pkg/controllers/services_controller.go b/pkg/controllers/services_controller.go index e030517..c13d919 100644 --- a/pkg/controllers/services_controller.go +++ b/pkg/controllers/services_controller.go @@ -3,6 +3,7 @@ package controllers import ( "context" "fmt" + "strings" "sync" "time" @@ -102,6 +103,123 @@ type ServicesController struct { // to backend pods hosted here. Empty disables the check and programs // everything everywhere. NodeName string + + // RetryInterval is how often failed datapath writes are re-attempted. + // Zero selects defaultRetryInterval. + RetryInterval time.Duration + + // retryMu guards pendingServices and pendingCleanup. + retryMu sync.Mutex + + // pendingServices holds the keys of services whose datapath programming + // failed. The informers only deliver events, so without a re-attempt a + // transient nftables failure leaves the service unprogrammed until the + // next event — long enough for a public IP to stay dark for minutes. + pendingServices map[string]struct{} + + // pendingCleanup records that the startup reconciliation failed and has + // to run again. It stays non-fatal, but it no longer waits for the next + // event or the 12-hour informer resync. + pendingCleanup bool +} + +// defaultRetryInterval is short enough that a failed write is repaired well +// within a human noticing, and long enough that a permanent failure does not +// flood the log. +const defaultRetryInterval = 30 * time.Second + +// markPending queues a service for another programming attempt. +func (c *ServicesController) markPending(namespace, name string) { + c.retryMu.Lock() + defer c.retryMu.Unlock() + if c.pendingServices == nil { + c.pendingServices = make(map[string]struct{}) + } + c.pendingServices[makeKey(namespace, name)] = struct{}{} +} + +// clearPending drops a service from the retry set after a clean pass. +func (c *ServicesController) clearPending(namespace, name string) { + c.retryMu.Lock() + defer c.retryMu.Unlock() + delete(c.pendingServices, makeKey(namespace, name)) +} + +// markCleanupPending queues the startup reconciliation for another attempt. +func (c *ServicesController) markCleanupPending() { + c.retryMu.Lock() + defer c.retryMu.Unlock() + c.pendingCleanup = true +} + +// takePending returns the queued work and clears it. Anything that fails again +// is re-queued by the attempt itself. +func (c *ServicesController) takePending() (services []string, cleanup bool) { + c.retryMu.Lock() + defer c.retryMu.Unlock() + for k := range c.pendingServices { + services = append(services, k) + } + c.pendingServices = nil + cleanup = c.pendingCleanup + c.pendingCleanup = false + return services, cleanup +} + +// retryPending re-attempts everything that failed since the last pass. Every +// datapath call is idempotent, so a re-attempt on something already correct is +// a no-op. +func (c *ServicesController) retryPending() { + services, cleanup := c.takePending() + + for _, key := range services { + ns, name, ok := splitKey(key) + if !ok { + continue + } + se, exists := c.Services.Get(ns, name) + if !exists || !hasValidServiceIP(se.Service) || !hasValidEndpointIP(se.Endpoint) { + // The service went away or lost its endpoint; the delete paths + // have already withdrawn its rules. + continue + } + log.Info("retrying datapath programming", "service", key) + c.applyRules(se.Service, se.Endpoint, "on retry") + } + + if cleanup { + log.Info("retrying startup cleanup") + if err := c.cleanupRemovedServices(); err != nil { + log.Error(err, "cleanup retry failed, will try again") + } + } +} + +// runRetryLoop re-attempts failed datapath writes until the context ends. +func (c *ServicesController) runRetryLoop(ctx context.Context) { + interval := c.RetryInterval + if interval <= 0 { + interval = defaultRetryInterval + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.retryPending() + } + } +} + +// splitKey reverses makeKey. +func splitKey(key string) (namespace, name string, ok bool) { + i := strings.Index(key, "/") + if i <= 0 || i == len(key)-1 { + return "", "", false + } + return key[:i], key[i+1:], true } // endpointNode returns the node hosting the endpoint's first address. @@ -156,20 +274,37 @@ func (c *ServicesController) applyRules(svc *v1.Service, ep *v1.Endpoints, ctx s // the last place where the pod IP can still be turned back into the // service IP the client's conntrack is waiting for. Without it the reply // arrives with the wrong source and the client answers it with a RST. + failed := false if err := c.Proxy.EnsureEgressSNAT(svcIP, podIP); err != nil { log.Error(err, "failed to ensure egress SNAT "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = true } // The destination rewrite and the port filter stay on the hosting node. if !c.servesEndpoint(ep) { c.withdrawIngressRules(svcIP, podIP, ctx+" (backend not on this node)") + c.recordOutcome(svc, failed) return } if err := c.Proxy.EnsureIngressDNAT(svcIP, podIP); err != nil { log.Error(err, "failed to ensure ingress DNAT "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = true + } + if err := c.reconcilePortFilter(svc, svcIP, podIP, ctx); err != nil { + failed = true + } + c.recordOutcome(svc, failed) +} + +// recordOutcome queues the service for another attempt when any part of its +// datapath failed, and clears a previous failure once a pass is clean. +func (c *ServicesController) recordOutcome(svc *v1.Service, failed bool) { + if failed { + c.markPending(svc.Namespace, svc.Name) + return } - c.reconcilePortFilter(svc, svcIP, podIP, ctx) + c.clearPending(svc.Namespace, svc.Name) } // withdrawIngressRules removes the ingress half only — destination rewrite and @@ -304,11 +439,17 @@ func (c *ServicesController) Start(ctx context.Context) error { // half-programmed, whereas the informers below converge on the next event. log.Info("running cleanup for removed services") if err := c.cleanupRemovedServices(); err != nil { - log.Error(err, "cleanup of removed services failed, continuing with reconciliation") + log.Error(err, "cleanup of removed services failed, queued for retry") + c.markCleanupPending() } else { log.Info("cleanup of removed services completed") } + // Re-attempt whatever failed. The informers only deliver events, so + // without this a transient nftables failure leaves a service + // unprogrammed until the next one. + go c.runRetryLoop(ctx) + <-ctx.Done() log.Info("shutting down services-controller") @@ -611,28 +752,39 @@ func allowICMP(svc *v1.Service) bool { // reconcilePortFilter applies the port-filter and ICMP-allow state implied by // the service's annotations. Call sites pass the resolved svcIP/podIP and a // short context string that ends up in error logs. -func (c *ServicesController) reconcilePortFilter(svc *v1.Service, svcIP, podIP, ctx string) { +// It returns an error when any part of the port-filter state could not be +// applied, so the caller can queue the service for another attempt. A filtered +// pod with missing allowed_ports drops every packet, so a silent failure here +// is an outage. +func (c *ServicesController) reconcilePortFilter(svc *v1.Service, svcIP, podIP, ctx string) error { + var failed error if wholeIPPassthrough(svc) { if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil { log.Error(err, "failed to delete port filter "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = err } if err := c.Proxy.DeleteICMPAllow(svcIP, podIP); err != nil { log.Error(err, "failed to delete ICMP allow "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = err } - return + return failed } if err := c.Proxy.EnsurePortFilter(svcIP, podIP, svc.Spec.Ports); err != nil { log.Error(err, "failed to ensure port filter "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = err } if allowICMP(svc) { if err := c.Proxy.EnsureICMPAllow(svcIP, podIP); err != nil { log.Error(err, "failed to ensure ICMP allow "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = err } } else { if err := c.Proxy.DeleteICMPAllow(svcIP, podIP); err != nil { log.Error(err, "failed to delete ICMP allow "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = err } } + return failed } // clearPortFilter unconditionally removes both port-filter and ICMP-allow diff --git a/pkg/controllers/services_controller_test.go b/pkg/controllers/services_controller_test.go index 46b2ff5..8e46e60 100644 --- a/pkg/controllers/services_controller_test.go +++ b/pkg/controllers/services_controller_test.go @@ -1,6 +1,7 @@ package controllers import ( + "errors" "testing" v1 "k8s.io/api/core/v1" @@ -40,10 +41,21 @@ type recordingProxy struct { calls []string keepEgress map[string]string keepIngress map[string]string + + // failEgress makes EnsureEgressSNAT fail, standing in for a refused + // nftables commit. + failEgress bool + // failPortFilter makes EnsurePortFilter fail. + failPortFilter bool + // failCleanup makes CleanupRules fail. + failCleanup bool } func (r *recordingProxy) EnsureEgressSNAT(svcIP, podIP string) error { r.calls = append(r.calls, "EnsureEgressSNAT") + if r.failEgress { + return errors.New("commit refused") + } return nil } @@ -66,11 +78,17 @@ func (r *recordingProxy) CleanupRules(keepEgress, keepIngress map[string]string) r.calls = append(r.calls, "CleanupRules") r.keepEgress = keepEgress r.keepIngress = keepIngress + if r.failCleanup { + return errors.New("commit refused") + } return nil } func (r *recordingProxy) EnsurePortFilter(svcIP, podIP string, ports []v1.ServicePort) error { r.calls = append(r.calls, "EnsurePortFilter") + if r.failPortFilter { + return errors.New("commit refused") + } return nil } @@ -294,3 +312,127 @@ func TestCleanupRemovedServicesScopesKeepMaps(t *testing.T) { t.Errorf("keepIngress = %v, want %v", rec.keepIngress, wantIngress) } } + +// A refused commit must not be forgotten: the informers only deliver events, so +// a service left unprogrammed would stay that way until the next one. +func TestFailedProgrammingIsQueuedForRetry(t *testing.T) { + svc := lbService("192.0.2.10", map[string]string{"networking.cozystack.io/wholeIP": "false"}) + svc.Namespace, svc.Name = "ns", "svc1" + ep := epOnNode("10.0.0.1", "node-a") + + px := &recordingProxy{failEgress: true} + ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} + ctrl.applyRules(svc, ep, "test") + + keys, _ := ctrl.takePending() + if len(keys) != 1 || keys[0] != "ns/svc1" { + t.Fatalf("failed write must be queued, got %v", keys) + } + + // A clean pass clears it again. + px.failEgress = false + ctrl.applyRules(svc, ep, "test") + keys, _ = ctrl.takePending() + if len(keys) != 0 { + t.Errorf("clean pass must clear the queue, got %v", keys) + } +} + +// A port filter that could not be applied is an outage — the pod is filtered +// with no open port — so it must be queued too. +func TestFailedPortFilterIsQueuedForRetry(t *testing.T) { + svc := lbService("192.0.2.10", map[string]string{"networking.cozystack.io/wholeIP": "false"}) + svc.Namespace, svc.Name = "ns", "svc2" + + px := &recordingProxy{failPortFilter: true} + ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} + ctrl.applyRules(svc, epOnNode("10.0.0.1", "node-a"), "test") + + keys, _ := ctrl.takePending() + if len(keys) != 1 || keys[0] != "ns/svc2" { + t.Fatalf("failed port filter must be queued, got %v", keys) + } +} + +// The retry pass must re-apply from the stored state. +func TestRetryPendingReappliesDatapath(t *testing.T) { + svc := lbService("192.0.2.10", map[string]string{"networking.cozystack.io/wholeIP": "false"}) + svc.Namespace, svc.Name = "ns", "svc3" + ep := epOnNode("10.0.0.1", "node-a") + + px := &recordingProxy{failEgress: true} + ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} + ctrl.Services = NewServiceMap() + ctrl.Services.Set("ns", "svc3", &ServiceEndpoints{Service: svc, Endpoint: ep}) + ctrl.applyRules(svc, ep, "test") + + px.failEgress = false + px.calls = nil + ctrl.retryPending() + + for _, want := range []string{"EnsureEgressSNAT", "EnsureIngressDNAT", "EnsurePortFilter"} { + if !px.has(want) { + t.Errorf("retry must call %s, got %v", want, px.calls) + } + } + if keys, _ := ctrl.takePending(); len(keys) != 0 { + t.Errorf("successful retry must clear the queue, got %v", keys) + } +} + +// A service that disappeared between the failure and the retry must not be +// re-applied; its delete path has already withdrawn the rules. +func TestRetryPendingSkipsVanishedService(t *testing.T) { + px := &recordingProxy{} + ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} + ctrl.Services = NewServiceMap() + ctrl.markPending("ns", "gone") + + ctrl.retryPending() + if len(px.calls) != 0 { + t.Errorf("vanished service must not be re-applied, got %v", px.calls) + } +} + +// A failed startup cleanup must be re-attempted rather than wait for the next +// event or the 12-hour informer resync. +func TestFailedCleanupIsQueuedAndRetried(t *testing.T) { + px := &recordingProxy{failCleanup: true} + ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} + ctrl.Services = NewServiceMap() + + if err := ctrl.cleanupRemovedServices(); err == nil { + t.Fatal("cleanupRemovedServices must surface the failure") + } + ctrl.markCleanupPending() + + px.failCleanup = false + px.calls = nil + ctrl.retryPending() + if !px.has("CleanupRules") { + t.Errorf("cleanup must be retried, got %v", px.calls) + } + if _, pending := ctrl.takePending(); pending { + t.Error("successful cleanup retry must clear the flag") + } +} + +func TestSplitKey(t *testing.T) { + cases := []struct { + key, ns, name string + ok bool + }{ + {"ns/name", "ns", "name", true}, + {"ns/sub/name", "ns", "sub/name", true}, + {"noslash", "", "", false}, + {"/name", "", "", false}, + {"ns/", "", "", false}, + {"", "", "", false}, + } + for _, c := range cases { + ns, name, ok := splitKey(c.key) + if ok != c.ok || ns != c.ns || name != c.name { + t.Errorf("splitKey(%q) = (%q,%q,%v), want (%q,%q,%v)", c.key, ns, name, ok, c.ns, c.name, c.ok) + } + } +} diff --git a/pkg/proxy/nft.go b/pkg/proxy/nft.go index 7cb296b..e66e2f4 100644 --- a/pkg/proxy/nft.go +++ b/pkg/proxy/nft.go @@ -585,6 +585,13 @@ func parsePair(svcIP, podIP string) (net.IP, net.IP, error) { // whole flush. Deletions must therefore be committed on their own, so a stale // element cannot mask a genuine failure among the additions that would // otherwise share the batch. +// Note on detection: the errno travels as an *netlink.OpError wrapping a +// syscall.Errno, which errors.Is unwraps, and that is the path taken when the +// kernel refuses a batch. github.com/google/nftables v0.3.0 has one branch in +// receiveAckAware that formats a trailing error ack with %v instead of wrapping +// it, and an ENOENT arriving that way reads as an ordinary failure. v0.3.0 is +// the latest release, so the caller is returned an error in that case rather +// than silently continuing — which is the safe direction. func (p *NFTProxyProcessor) flushTolerateENOENT(op string) error { err := p.conn.Flush() if err == nil { @@ -711,7 +718,10 @@ func (p *NFTProxyProcessor) CleanupRules(keepEgress, keepIngress map[string]stri // Startup cleanup must not be fatal: aborting here takes the whole // DaemonSet pod down and leaves the node's datapath half-programmed, while // the reconcile loop would have converged on the next event anyway. - if err := p.flushTolerateENOENT("CleanupRules additions"); err != nil { + // Additions are committed strictly: an ENOENT on an addition-only batch + // means the table or the set is gone, so tolerating it would report a + // successful reconciliation with nothing installed. + if err := p.conn.Flush(); err != nil { log.Error(err, "Failed to commit cleanup changes") return fmt.Errorf("failed to commit cleanup changes: %v", err) } @@ -967,8 +977,10 @@ func (p *NFTProxyProcessor) CleanupPortFilters(keep map[string]PortFilterEntry) } } - // 5. Commit the additions. Startup cleanup must not abort the pod. - if err := p.flushTolerateENOENT("CleanupPortFilters additions"); err != nil { + // 5. Commit the additions strictly: an ENOENT here means the table or the + // set is missing, not that an element was already gone. The caller keeps + // startup non-fatal. + if err := p.conn.Flush(); err != nil { return fmt.Errorf("failed to flush CleanupPortFilters: %v", err) } log.Info("CleanupPortFilters completed", @@ -1114,7 +1126,8 @@ func (p *NFTProxyProcessor) CleanupICMPAllow(keep map[string]string) error { } } - if err := p.flushTolerateENOENT("CleanupICMPAllow additions"); err != nil { + // Additions committed strictly, see CleanupPortFilters. + if err := p.conn.Flush(); err != nil { return fmt.Errorf("failed to flush CleanupICMPAllow: %v", err) } log.Info("CleanupICMPAllow completed", From 2eafd9a44ed8c1dfe95a3ef1bac2d25f31853b11 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 15 Sep 2026 15:01:55 +0200 Subject: [PATCH 07/42] controllers: serialize whole reconciliations against the retry loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retry loop added in the previous commit is a third concurrent reconciler, and it read the stored pair the way the informer callbacks do: ServiceMap.Get hands back the pointer to the shared ServiceEndpoints, whose Endpoint field SetEndpoint rewrites under the lock. Reading that field once Get has returned is an unsynchronized read, and the race detector says so. The consequence is worse than the read itself. An informer callback withdraws the rules of a replaced endpoint and then applies the new one. The retry could slip between the two carrying the endpoint it had snapshotted, and its writes would land after the update — restoring the mapping of a pod that no longer exists and pointing the service IP at a dead backend. The proxy serializes individual operations, not a whole reconciliation, so nothing prevented that ordering. Add ServiceMap.Snapshot, which copies the pair out under the lock, and a reconcileMu held across each whole reconciliation: the six informer callbacks, the startup cleanup, and one retry item at a time. The retry takes and releases it per item rather than for the whole pass, so a long pass does not starve the callbacks. The lock is always taken before the proxy's own, never the other way round. The concurrency test drives endpoint replacement against the retry loop; reverting either half of this fix makes it report a data race. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5 (1M context) --- pkg/controllers/services_controller.go | 81 ++++++++++++++++++-- pkg/controllers/services_controller_test.go | 83 +++++++++++++++++++-- 2 files changed, 149 insertions(+), 15 deletions(-) diff --git a/pkg/controllers/services_controller.go b/pkg/controllers/services_controller.go index c13d919..81a6bf9 100644 --- a/pkg/controllers/services_controller.go +++ b/pkg/controllers/services_controller.go @@ -56,6 +56,22 @@ func (sm *ServiceMap) Get(namespace, name string) (*ServiceEndpoints, bool) { return se, ok } +// Snapshot returns the stored Service and Endpoints under the lock. +// +// Get hands back the pointer to the shared ServiceEndpoints, whose Endpoint +// field SetEndpoint rewrites under the lock. Reading that field after Get has +// returned is an unsynchronized read, so any caller outside the informer +// callbacks has to come through here. +func (sm *ServiceMap) Snapshot(namespace, name string) (*v1.Service, *v1.Endpoints, bool) { + sm.mu.Lock() + defer sm.mu.Unlock() + se, ok := sm.serviceMapping[makeKey(namespace, name)] + if !ok || se == nil { + return nil, nil, false + } + return se.Service, se.Endpoint, true +} + // Set stores the ServiceEndpoints under the given namespace and name. func (sm *ServiceMap) Set(namespace, name string, se *ServiceEndpoints) { sm.mu.Lock() @@ -108,6 +124,16 @@ type ServicesController struct { // Zero selects defaultRetryInterval. RetryInterval time.Duration + // reconcileMu serializes whole reconciliations, not just the individual + // datapath calls the proxy already serializes. + // + // An informer callback withdraws the rules of a replaced endpoint and then + // applies the new one. Without this lock the retry goroutine can slip + // between the two with the endpoint it snapshotted a moment earlier, and + // its writes land after the update — restoring the mapping of a pod that + // is gone, which points the service IP at a dead backend. + reconcileMu sync.Mutex + // retryMu guards pendingServices and pendingCleanup. retryMu sync.Mutex @@ -177,14 +203,7 @@ func (c *ServicesController) retryPending() { if !ok { continue } - se, exists := c.Services.Get(ns, name) - if !exists || !hasValidServiceIP(se.Service) || !hasValidEndpointIP(se.Endpoint) { - // The service went away or lost its endpoint; the delete paths - // have already withdrawn its rules. - continue - } - log.Info("retrying datapath programming", "service", key) - c.applyRules(se.Service, se.Endpoint, "on retry") + c.retryOne(ns, name, key) } if cleanup { @@ -195,6 +214,24 @@ func (c *ServicesController) retryPending() { } } +// retryOne re-applies a single service, reading its state and programming the +// datapath under the reconciliation lock so an informer callback cannot +// interleave and have this attempt restore what it just withdrew. +func (c *ServicesController) retryOne(namespace, name, key string) { + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + + svc, ep, exists := c.Services.Snapshot(namespace, name) + if !exists || !hasValidServiceIP(svc) || !hasValidEndpointIP(ep) { + // The service went away or lost its endpoint; the delete paths have + // already withdrawn its rules. Drop it from the queue. + c.clearPending(namespace, name) + return + } + log.Info("retrying datapath programming", "service", key) + c.applyRules(svc, ep, "on retry") +} + // runRetryLoop re-attempts failed datapath writes until the context ends. func (c *ServicesController) runRetryLoop(ctx context.Context) { interval := c.RetryInterval @@ -458,6 +495,10 @@ func (c *ServicesController) Start(ctx context.Context) error { // addServiceFunc handles the addition of a service. func (c *ServicesController) addServiceFunc(obj interface{}) { + // Reconciliations are serialized end to end, see reconcileMu. + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + svc, ok := obj.(*v1.Service) if !ok { // Object is not a Service. @@ -490,6 +531,10 @@ func (c *ServicesController) addServiceFunc(obj interface{}) { // deleteServiceFunc handles the deletion of a service. func (c *ServicesController) deleteServiceFunc(obj interface{}) { + // Reconciliations are serialized end to end, see reconcileMu. + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + svc, ok := obj.(*v1.Service) if !ok { // object is not Service @@ -513,6 +558,10 @@ func (c *ServicesController) deleteServiceFunc(obj interface{}) { // updateServiceFunc handles service updates. func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) { + // Reconciliations are serialized end to end, see reconcileMu. + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + // Cast the object to a Service type. svc, ok := newObj.(*v1.Service) if !ok { @@ -588,6 +637,10 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) { // addEndpointFunc handles the addition of endpoints. func (c *ServicesController) addEndpointFunc(obj interface{}) { + // Reconciliations are serialized end to end, see reconcileMu. + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + // Cast the object to an Endpoints type. ep, ok := obj.(*v1.Endpoints) if !ok { @@ -614,6 +667,10 @@ func (c *ServicesController) addEndpointFunc(obj interface{}) { // deleteEndpointFunc handles endpoint deletions. func (c *ServicesController) deleteEndpointFunc(obj interface{}) { + // Reconciliations are serialized end to end, see reconcileMu. + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + ep, ok := obj.(*v1.Endpoints) if !ok { // object is not Endpoints @@ -637,6 +694,10 @@ func (c *ServicesController) deleteEndpointFunc(obj interface{}) { // updateEndpointFunc handles updates to endpoints. func (c *ServicesController) updateEndpointFunc(oldObj, newObj interface{}) { + // Reconciliations are serialized end to end, see reconcileMu. + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + ep, ok := newObj.(*v1.Endpoints) if !ok { // object is not Endpoints @@ -800,6 +861,10 @@ func (c *ServicesController) clearPortFilter(svcIP, podIP, ctx string) { // cleanupRemovedServices performs an initial cleanup for removed services. func (c *ServicesController) cleanupRemovedServices() error { + // Reconciliations are serialized end to end, see reconcileMu. + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + // keepEgress holds every managed pair, because the source rewrite is // programmed cluster-wide. keepIngress holds only the pairs whose backend // runs here, because the destination rewrite is node-local. The difference diff --git a/pkg/controllers/services_controller_test.go b/pkg/controllers/services_controller_test.go index 8e46e60..03f388d 100644 --- a/pkg/controllers/services_controller_test.go +++ b/pkg/controllers/services_controller_test.go @@ -2,7 +2,10 @@ package controllers import ( "errors" + "fmt" + "sync" "testing" + "time" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,6 +41,7 @@ func lbService(svcIP string, annot map[string]string) *v1.Service { // recordingProxy captures which datapath calls a controller makes. type recordingProxy struct { nat.DummyProxyProcessor + mu sync.Mutex calls []string keepEgress map[string]string keepIngress map[string]string @@ -52,7 +56,7 @@ type recordingProxy struct { } func (r *recordingProxy) EnsureEgressSNAT(svcIP, podIP string) error { - r.calls = append(r.calls, "EnsureEgressSNAT") + r.record("EnsureEgressSNAT") if r.failEgress { return errors.New("commit refused") } @@ -60,22 +64,22 @@ func (r *recordingProxy) EnsureEgressSNAT(svcIP, podIP string) error { } func (r *recordingProxy) DeleteEgressSNAT(svcIP, podIP string) error { - r.calls = append(r.calls, "DeleteEgressSNAT") + r.record("DeleteEgressSNAT") return nil } func (r *recordingProxy) EnsureIngressDNAT(svcIP, podIP string) error { - r.calls = append(r.calls, "EnsureIngressDNAT") + r.record("EnsureIngressDNAT") return nil } func (r *recordingProxy) DeleteIngressDNAT(svcIP, podIP string) error { - r.calls = append(r.calls, "DeleteIngressDNAT") + r.record("DeleteIngressDNAT") return nil } func (r *recordingProxy) CleanupRules(keepEgress, keepIngress map[string]string) error { - r.calls = append(r.calls, "CleanupRules") + r.record("CleanupRules") r.keepEgress = keepEgress r.keepIngress = keepIngress if r.failCleanup { @@ -85,7 +89,7 @@ func (r *recordingProxy) CleanupRules(keepEgress, keepIngress map[string]string) } func (r *recordingProxy) EnsurePortFilter(svcIP, podIP string, ports []v1.ServicePort) error { - r.calls = append(r.calls, "EnsurePortFilter") + r.record("EnsurePortFilter") if r.failPortFilter { return errors.New("commit refused") } @@ -93,11 +97,21 @@ func (r *recordingProxy) EnsurePortFilter(svcIP, podIP string, ports []v1.Servic } func (r *recordingProxy) DeletePortFilter(svcIP, podIP string) error { - r.calls = append(r.calls, "DeletePortFilter") + r.record("DeletePortFilter") return nil } +// record appends under the lock: the retry loop and the informer callbacks +// both drive the proxy in the concurrency test. +func (r *recordingProxy) record(call string) { + r.mu.Lock() + defer r.mu.Unlock() + r.calls = append(r.calls, call) +} + func (r *recordingProxy) has(call string) bool { + r.mu.Lock() + defer r.mu.Unlock() for _, c := range r.calls { if c == call { return true @@ -436,3 +450,58 @@ func TestSplitKey(t *testing.T) { } } } + +// The retry goroutine and the informer callbacks reconcile the same stored +// pair. The retry must read it under the map lock and hold the reconciliation +// lock while it programs, or it applies an endpoint the update has already +// withdrawn — pointing the service IP at a pod that is gone. +// +// Run with -race: an unsynchronized read of the shared Endpoint field shows up +// here, which is what reading it through ServiceMap.Get used to do. +func TestRetryIsSerializedAgainstEndpointUpdates(t *testing.T) { + svc := lbService("192.0.2.10", map[string]string{"networking.cozystack.io/wholeIP": "false"}) + svc.Namespace, svc.Name = "ns", "svc" + + ctrl := &ServicesController{Proxy: &recordingProxy{}, NodeName: "node-a"} + ctrl.Services = NewServiceMap() + ctrl.Services.Set("ns", "svc", &ServiceEndpoints{ + Service: svc, + Endpoint: epOnNode("10.0.0.1", "node-a"), + }) + + stop := make(chan struct{}) + var wg sync.WaitGroup + + // Informer side: keep replacing the endpoint, as a migrating VM does. + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + ctrl.Services.SetEndpoint("ns", "svc", epOnNode(fmt.Sprintf("10.0.0.%d", i%250+1), "node-a")) + } + }() + + // Retry side. + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + ctrl.markPending("ns", "svc") + ctrl.retryPending() + } + }() + + time.Sleep(150 * time.Millisecond) + close(stop) + wg.Wait() +} From 0b10c267672d811316f6276c9f9a9a03f55758ab Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 15 Sep 2026 15:27:59 +0200 Subject: [PATCH 08/42] controllers: retry failed datapath withdrawals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retry queue only covered programming. A failed removal was logged and dropped, and it could not go through the service queue either: by the time a withdrawal runs the service is usually gone from the map, so there is nothing left to re-derive the pair from. Leaving that state behind is not merely untidy. A stale svc_pod entry keeps translating a service IP towards a pod that no longer exists, and a stale pod_svc entry rewrites the source of whatever pod next receives that IP — on a shared /16 with pod IPs recycling, one tenant's egress leaving under another tenant's service IP. A stale filtered_pods entry applies someone else's port list to the new occupant. Queue the pair explicitly instead. clearPortFilter, withdrawIngressRules and withdrawRules now return an error, the two withdraw paths queue themselves on failure, and the retry pass re-attempts them. A full withdrawal supersedes an ingress-only one for the same pair, so a node that also has to drop the source rewrite does not lose that. Withdrawals are retried before the applies, and programming a pair clears any removal queued for it, so an endpoint that flaps back to the pod IP whose withdrawal failed does not have its freshly installed rules deleted by the retry. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5 (1M context) --- pkg/controllers/services_controller.go | 118 ++++++++++++++++++-- pkg/controllers/services_controller_test.go | 98 +++++++++++++++- 2 files changed, 202 insertions(+), 14 deletions(-) diff --git a/pkg/controllers/services_controller.go b/pkg/controllers/services_controller.go index 81a6bf9..dba9508 100644 --- a/pkg/controllers/services_controller.go +++ b/pkg/controllers/services_controller.go @@ -143,12 +143,60 @@ type ServicesController struct { // next event — long enough for a public IP to stay dark for minutes. pendingServices map[string]struct{} + // pendingWithdrawals holds datapath state that could not be removed. + // + // A failed withdrawal cannot go through pendingServices: by then the + // service is usually gone from Services, so there is nothing to re-derive + // the pair from. It is carried explicitly instead. Leaving it behind is + // not merely untidy — a stale pod_svc entry rewrites the source of + // whatever pod next receives that IP, which on a shared /16 means one + // tenant's egress leaving under another tenant's service IP. + pendingWithdrawals map[string]withdrawal + // pendingCleanup records that the startup reconciliation failed and has // to run again. It stays non-fatal, but it no longer waits for the next // event or the 12-hour informer resync. pendingCleanup bool } +// withdrawal is datapath state waiting to be removed. +type withdrawal struct { + svcIP, podIP string + // egress also withdraws the source rewrite. False when only the ingress + // half has to go, which is the case on a node that stopped hosting the + // backend but still carries the cluster-wide source rewrite. + egress bool +} + +// withdrawalKey identifies a pair independently of which half is pending. +func withdrawalKey(svcIP, podIP string) string { return svcIP + "/" + podIP } + +// markWithdrawal queues datapath state for another removal attempt. +func (c *ServicesController) markWithdrawal(svcIP, podIP string, egress bool) { + c.retryMu.Lock() + defer c.retryMu.Unlock() + if c.pendingWithdrawals == nil { + c.pendingWithdrawals = make(map[string]withdrawal) + } + k := withdrawalKey(svcIP, podIP) + // A full withdrawal supersedes an ingress-only one for the same pair. + if prev, ok := c.pendingWithdrawals[k]; ok && prev.egress { + egress = true + } + c.pendingWithdrawals[k] = withdrawal{svcIP: svcIP, podIP: podIP, egress: egress} +} + +// clearWithdrawal drops a queued removal. +// +// Called when the same pair is programmed again — an endpoint that flapped back +// to the pod IP whose withdrawal failed. Without this the retry would delete +// the rules that were just reinstalled. +func (c *ServicesController) clearWithdrawal(svcIP, podIP string) { + c.retryMu.Lock() + defer c.retryMu.Unlock() + delete(c.pendingWithdrawals, withdrawalKey(svcIP, podIP)) +} + // defaultRetryInterval is short enough that a failed write is repaired well // within a human noticing, and long enough that a permanent failure does not // flood the log. @@ -180,23 +228,33 @@ func (c *ServicesController) markCleanupPending() { // takePending returns the queued work and clears it. Anything that fails again // is re-queued by the attempt itself. -func (c *ServicesController) takePending() (services []string, cleanup bool) { +func (c *ServicesController) takePending() (services []string, withdrawals []withdrawal, cleanup bool) { c.retryMu.Lock() defer c.retryMu.Unlock() for k := range c.pendingServices { services = append(services, k) } c.pendingServices = nil + for _, w := range c.pendingWithdrawals { + withdrawals = append(withdrawals, w) + } + c.pendingWithdrawals = nil cleanup = c.pendingCleanup c.pendingCleanup = false - return services, cleanup + return services, withdrawals, cleanup } // retryPending re-attempts everything that failed since the last pass. Every // datapath call is idempotent, so a re-attempt on something already correct is // a no-op. func (c *ServicesController) retryPending() { - services, cleanup := c.takePending() + services, withdrawals, cleanup := c.takePending() + + // Withdrawals first: an apply for the same pair clears its queued removal + // at the moment it succeeds, so it wins either way. + for _, w := range withdrawals { + c.retryWithdrawal(w) + } for _, key := range services { ns, name, ok := splitKey(key) @@ -214,6 +272,21 @@ func (c *ServicesController) retryPending() { } } +// retryWithdrawal re-attempts a removal that failed. The pair is carried +// explicitly because the service it belonged to is usually gone by now. +func (c *ServicesController) retryWithdrawal(w withdrawal) { + c.reconcileMu.Lock() + defer c.reconcileMu.Unlock() + + log.Info("retrying datapath withdrawal", "svcIP", w.svcIP, "podIP", w.podIP, "egress", w.egress) + if w.egress { + // Re-queues itself on failure. + _ = c.withdrawRules(w.svcIP, w.podIP, "on retry") + return + } + _ = c.withdrawIngressRules(w.svcIP, w.podIP, "on retry") +} + // retryOne re-applies a single service, reading its state and programming the // datapath under the reconciliation lock so an informer callback cannot // interleave and have this attempt restore what it just withdrew. @@ -311,6 +384,9 @@ func (c *ServicesController) applyRules(svc *v1.Service, ep *v1.Endpoints, ctx s // the last place where the pod IP can still be turned back into the // service IP the client's conntrack is waiting for. Without it the reply // arrives with the wrong source and the client answers it with a RST. + // This pair is wanted again; drop any removal still queued for it. + c.clearWithdrawalFor(svc, ep) + failed := false if err := c.Proxy.EnsureEgressSNAT(svcIP, podIP); err != nil { log.Error(err, "failed to ensure egress SNAT "+ctx, "svcIP", svcIP, "podIP", podIP) @@ -319,7 +395,9 @@ func (c *ServicesController) applyRules(svc *v1.Service, ep *v1.Endpoints, ctx s // The destination rewrite and the port filter stay on the hosting node. if !c.servesEndpoint(ep) { - c.withdrawIngressRules(svcIP, podIP, ctx+" (backend not on this node)") + if err := c.withdrawIngressRules(svcIP, podIP, ctx+" (backend not on this node)"); err != nil { + failed = true + } c.recordOutcome(svc, failed) return } @@ -344,24 +422,42 @@ func (c *ServicesController) recordOutcome(svc *v1.Service, failed bool) { c.clearPending(svc.Namespace, svc.Name) } +// clearWithdrawalFor drops a queued removal for a pair that has just been +// programmed again, so the retry does not delete what was reinstalled. +func (c *ServicesController) clearWithdrawalFor(svc *v1.Service, ep *v1.Endpoints) { + c.clearWithdrawal(svc.Status.LoadBalancer.Ingress[0].IP, ep.Subsets[0].Addresses[0].IP) +} + // withdrawIngressRules removes the ingress half only — destination rewrite and // port filter — and leaves the source rewrite in place. This is what a node // that does not host the backend must end up with. -func (c *ServicesController) withdrawIngressRules(svcIP, podIP, ctx string) { - c.clearPortFilter(svcIP, podIP, ctx) +// It returns an error when any part could not be removed, so the caller can +// queue the pair for another attempt. +func (c *ServicesController) withdrawIngressRules(svcIP, podIP, ctx string) error { + failed := c.clearPortFilter(svcIP, podIP, ctx) if err := c.Proxy.DeleteIngressDNAT(svcIP, podIP); err != nil { log.Error(err, "failed to delete ingress DNAT "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = err + } + if failed != nil { + c.markWithdrawal(svcIP, podIP, false) } + return failed } // withdrawRules removes every datapath entry for the pair, both halves. Used // when the pair itself is going away: service deleted, endpoint gone, or a pod // IP that has been replaced. Absent entries are not an error. -func (c *ServicesController) withdrawRules(svcIP, podIP, ctx string) { - c.withdrawIngressRules(svcIP, podIP, ctx) +func (c *ServicesController) withdrawRules(svcIP, podIP, ctx string) error { + failed := c.withdrawIngressRules(svcIP, podIP, ctx) if err := c.Proxy.DeleteEgressSNAT(svcIP, podIP); err != nil { log.Error(err, "failed to delete egress SNAT "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = err + } + if failed != nil { + c.markWithdrawal(svcIP, podIP, true) } + return failed } // withdrawStaleEndpoint drops the rules of a previous endpoint whose pod IP no @@ -850,13 +946,17 @@ func (c *ServicesController) reconcilePortFilter(svc *v1.Service, svcIP, podIP, // clearPortFilter unconditionally removes both port-filter and ICMP-allow // state for (svcIP, podIP). Used by delete paths. -func (c *ServicesController) clearPortFilter(svcIP, podIP, ctx string) { +func (c *ServicesController) clearPortFilter(svcIP, podIP, ctx string) error { + var failed error if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil { log.Error(err, "failed to delete port filter "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = err } if err := c.Proxy.DeleteICMPAllow(svcIP, podIP); err != nil { log.Error(err, "failed to delete ICMP allow "+ctx, "svcIP", svcIP, "podIP", podIP) + failed = err } + return failed } // cleanupRemovedServices performs an initial cleanup for removed services. diff --git a/pkg/controllers/services_controller_test.go b/pkg/controllers/services_controller_test.go index 03f388d..6233c91 100644 --- a/pkg/controllers/services_controller_test.go +++ b/pkg/controllers/services_controller_test.go @@ -53,6 +53,10 @@ type recordingProxy struct { failPortFilter bool // failCleanup makes CleanupRules fail. failCleanup bool + // failDeleteEgress makes DeleteEgressSNAT fail. + failDeleteEgress bool + // failDeleteIngress makes DeleteIngressDNAT fail. + failDeleteIngress bool } func (r *recordingProxy) EnsureEgressSNAT(svcIP, podIP string) error { @@ -65,6 +69,9 @@ func (r *recordingProxy) EnsureEgressSNAT(svcIP, podIP string) error { func (r *recordingProxy) DeleteEgressSNAT(svcIP, podIP string) error { r.record("DeleteEgressSNAT") + if r.failDeleteEgress { + return errors.New("commit refused") + } return nil } @@ -75,6 +82,9 @@ func (r *recordingProxy) EnsureIngressDNAT(svcIP, podIP string) error { func (r *recordingProxy) DeleteIngressDNAT(svcIP, podIP string) error { r.record("DeleteIngressDNAT") + if r.failDeleteIngress { + return errors.New("commit refused") + } return nil } @@ -338,7 +348,7 @@ func TestFailedProgrammingIsQueuedForRetry(t *testing.T) { ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} ctrl.applyRules(svc, ep, "test") - keys, _ := ctrl.takePending() + keys, _, _ := ctrl.takePending() if len(keys) != 1 || keys[0] != "ns/svc1" { t.Fatalf("failed write must be queued, got %v", keys) } @@ -346,7 +356,7 @@ func TestFailedProgrammingIsQueuedForRetry(t *testing.T) { // A clean pass clears it again. px.failEgress = false ctrl.applyRules(svc, ep, "test") - keys, _ = ctrl.takePending() + keys, _, _ = ctrl.takePending() if len(keys) != 0 { t.Errorf("clean pass must clear the queue, got %v", keys) } @@ -362,7 +372,7 @@ func TestFailedPortFilterIsQueuedForRetry(t *testing.T) { ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} ctrl.applyRules(svc, epOnNode("10.0.0.1", "node-a"), "test") - keys, _ := ctrl.takePending() + keys, _, _ := ctrl.takePending() if len(keys) != 1 || keys[0] != "ns/svc2" { t.Fatalf("failed port filter must be queued, got %v", keys) } @@ -389,7 +399,7 @@ func TestRetryPendingReappliesDatapath(t *testing.T) { t.Errorf("retry must call %s, got %v", want, px.calls) } } - if keys, _ := ctrl.takePending(); len(keys) != 0 { + if keys, _, _ := ctrl.takePending(); len(keys) != 0 { t.Errorf("successful retry must clear the queue, got %v", keys) } } @@ -426,7 +436,7 @@ func TestFailedCleanupIsQueuedAndRetried(t *testing.T) { if !px.has("CleanupRules") { t.Errorf("cleanup must be retried, got %v", px.calls) } - if _, pending := ctrl.takePending(); pending { + if _, _, pending := ctrl.takePending(); pending { t.Error("successful cleanup retry must clear the flag") } } @@ -505,3 +515,81 @@ func TestRetryIsSerializedAgainstEndpointUpdates(t *testing.T) { close(stop) wg.Wait() } + +// A withdrawal that failed cannot go through the service queue: by then the +// service is gone from the map. Left behind, a stale pod_svc entry rewrites the +// source of whatever pod next receives that IP. +func TestFailedWithdrawalIsQueuedAndRetried(t *testing.T) { + px := &recordingProxy{failDeleteEgress: true} + ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} + ctrl.Services = NewServiceMap() + + if err := ctrl.withdrawRules("192.0.2.10", "10.0.0.1", "test"); err == nil { + t.Fatal("withdrawRules must surface the failure") + } + _, pending, _ := ctrl.takePending() + if len(pending) != 1 || pending[0].svcIP != "192.0.2.10" || !pending[0].egress { + t.Fatalf("failed withdrawal must be queued as a full one, got %+v", pending) + } + + // Re-queue it and let the retry succeed. + ctrl.markWithdrawal("192.0.2.10", "10.0.0.1", true) + px.failDeleteEgress = false + px.calls = nil + ctrl.retryPending() + + for _, want := range []string{"DeleteIngressDNAT", "DeleteEgressSNAT"} { + if !px.has(want) { + t.Errorf("retry must call %s, got %v", want, px.calls) + } + } + if _, pending, _ := ctrl.takePending(); len(pending) != 0 { + t.Errorf("successful retry must clear the queue, got %+v", pending) + } +} + +// An endpoint that flaps back to the pod IP whose withdrawal failed must not +// have its freshly installed rules deleted by the queued removal. +func TestReprogrammingClearsQueuedWithdrawal(t *testing.T) { + svc := lbService("192.0.2.10", map[string]string{"networking.cozystack.io/wholeIP": "false"}) + svc.Namespace, svc.Name = "ns", "svc" + ep := epOnNode("10.0.0.1", "node-a") + + px := &recordingProxy{} + ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} + ctrl.Services = NewServiceMap() + ctrl.markWithdrawal("192.0.2.10", "10.0.0.1", true) + + ctrl.applyRules(svc, ep, "test") + + if _, pending, _ := ctrl.takePending(); len(pending) != 0 { + t.Errorf("programming the pair again must drop its queued removal, got %+v", pending) + } +} + +// A node that stops hosting the backend withdraws only the ingress half, and +// must keep the cluster-wide source rewrite. +func TestIngressOnlyWithdrawalIsQueuedWithoutEgress(t *testing.T) { + px := &recordingProxy{failDeleteIngress: true} + ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} + + if err := ctrl.withdrawIngressRules("192.0.2.10", "10.0.0.1", "test"); err == nil { + t.Fatal("withdrawIngressRules must surface the failure") + } + _, pending, _ := ctrl.takePending() + if len(pending) != 1 || pending[0].egress { + t.Fatalf("ingress-only failure must not queue an egress withdrawal, got %+v", pending) + } +} + +// A full withdrawal supersedes an ingress-only one already queued for the pair. +func TestFullWithdrawalSupersedesIngressOnly(t *testing.T) { + ctrl := &ServicesController{Proxy: &recordingProxy{}} + ctrl.markWithdrawal("192.0.2.10", "10.0.0.1", true) + ctrl.markWithdrawal("192.0.2.10", "10.0.0.1", false) + + _, pending, _ := ctrl.takePending() + if len(pending) != 1 || !pending[0].egress { + t.Fatalf("full withdrawal must win, got %+v", pending) + } +} From c78640c707fb767cbe1fa6f6d2c7839d24edb77d Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 15 Sep 2026 16:04:23 +0200 Subject: [PATCH 09/42] controllers: build the startup snapshot from the informer stores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found on a lab restart, not by reading the code: the startup purge marked four live pod_svc mappings stale and deleted them, then reported podSvcAdded: 0. The datapath was only correct again because the endpoint callbacks happened to fire a second time afterwards. WaitForCacheSync returns once the store is populated, not once every initial callback has run. cleanupRemovedServices read Services, which those callbacks fill, so it computed its desired state from a half filled map and treated everything missing from it as stale — including the cluster-wide source rewrites belonging to backends on other nodes. Nothing guarantees a second callback: a service whose mapping was deleted that way waits for an event, or for the 12-hour resync. Read the informer stores instead, which are authoritative as soon as the sync returns. Services stays the fallback when no informer is attached, which is how the unit tests drive the controller. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5 (1M context) --- pkg/controllers/services_controller.go | 46 +++++++++++++- pkg/controllers/services_controller_test.go | 66 +++++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/services_controller.go b/pkg/controllers/services_controller.go index dba9508..31a218b 100644 --- a/pkg/controllers/services_controller.go +++ b/pkg/controllers/services_controller.go @@ -120,6 +120,15 @@ type ServicesController struct { // everything everywhere. NodeName string + // svcStore and epStore are the informer stores. The startup snapshot is + // built from them rather than from Services, because WaitForCacheSync + // returns once the store is populated, not once every initial callback + // has run. Reading the half-filled map made the purge treat a live + // mapping as stale and delete it, leaving the node without it until an + // event happened to re-apply that service — or until the 12-hour resync. + svcStore cache.Store + epStore cache.Store + // RetryInterval is how often failed datapath writes are re-attempted. // Zero selects defaultRetryInterval. RetryInterval time.Duration @@ -509,6 +518,8 @@ func (c *ServicesController) Start(ctx context.Context) error { }, ) + c.svcStore = serviceInformer.GetStore() + serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.addServiceFunc, DeleteFunc: c.deleteServiceFunc, @@ -551,6 +562,8 @@ func (c *ServicesController) Start(ctx context.Context) error { }, ) + c.epStore = endpointsInformer.GetStore() + endpointsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.addEndpointFunc, DeleteFunc: c.deleteEndpointFunc, @@ -959,6 +972,36 @@ func (c *ServicesController) clearPortFilter(svcIP, podIP, ctx string) error { return failed } +// snapshotSource returns the pairs the datapath should hold. +// +// It reads the informer stores when they are available, because they are +// authoritative as soon as WaitForCacheSync returns, whereas Services is +// filled by callbacks that may not have run yet. Falling back to Services +// keeps the controller usable without informers, which is how the unit tests +// drive it. +func (c *ServicesController) snapshotSource() map[string]*ServiceEndpoints { + if c.svcStore == nil || c.epStore == nil { + return c.Services.GetAll() + } + out := make(map[string]*ServiceEndpoints) + for _, obj := range c.svcStore.List() { + svc, ok := obj.(*v1.Service) + if !ok || !isCozyProxyService(svc) || !hasValidServiceIP(svc) { + continue + } + epObj, exists, err := c.epStore.GetByKey(makeKey(svc.Namespace, svc.Name)) + if err != nil || !exists { + continue + } + ep, ok := epObj.(*v1.Endpoints) + if !ok || !hasValidEndpointIP(ep) { + continue + } + out[makeKey(svc.Namespace, svc.Name)] = &ServiceEndpoints{Service: svc, Endpoint: ep} + } + return out +} + // cleanupRemovedServices performs an initial cleanup for removed services. func (c *ServicesController) cleanupRemovedServices() error { // Reconciliations are serialized end to end, see reconcileMu. @@ -972,8 +1015,7 @@ func (c *ServicesController) cleanupRemovedServices() error { // scoped both maps alike. keepEgress := make(map[string]string) keepIngress := make(map[string]string) - // Get a snapshot of all managed services. - allServices := c.Services.GetAll() + allServices := c.snapshotSource() for _, serviceEndpoints := range allServices { if serviceEndpoints.Service == nil || serviceEndpoints.Endpoint == nil { continue diff --git a/pkg/controllers/services_controller_test.go b/pkg/controllers/services_controller_test.go index 6233c91..4eaa39f 100644 --- a/pkg/controllers/services_controller_test.go +++ b/pkg/controllers/services_controller_test.go @@ -10,6 +10,8 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + nat "github.com/cozystack/cozy-proxy/pkg/proxy" ) @@ -593,3 +595,67 @@ func TestFullWithdrawalSupersedesIngressOnly(t *testing.T) { t.Fatalf("full withdrawal must win, got %+v", pending) } } + +// The startup snapshot must come from the informer stores, not from the map the +// callbacks fill. WaitForCacheSync returns once the store is populated, not +// once every initial callback has run, and reading the half-filled map made the +// purge delete a live mapping it then had to wait for an event to restore. +func TestSnapshotSourcePrefersInformerStores(t *testing.T) { + managed := lbService("192.0.2.10", nil) + managed.Namespace, managed.Name = "ns", "managed" + managed.Labels = map[string]string{"service.kubernetes.io/service-proxy-name": "cozy-proxy"} + + unmanaged := lbService("192.0.2.11", nil) + unmanaged.Namespace, unmanaged.Name = "ns", "unmanaged" + + noIP := &v1.Service{ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", Name: "noip", + Labels: map[string]string{"service.kubernetes.io/service-proxy-name": "cozy-proxy"}, + }} + + noEndpoint := lbService("192.0.2.12", nil) + noEndpoint.Namespace, noEndpoint.Name = "ns", "noep" + noEndpoint.Labels = map[string]string{"service.kubernetes.io/service-proxy-name": "cozy-proxy"} + + svcStore := cache.NewStore(cache.MetaNamespaceKeyFunc) + for _, s := range []*v1.Service{managed, unmanaged, noIP, noEndpoint} { + if err := svcStore.Add(s); err != nil { + t.Fatalf("seeding service store: %v", err) + } + } + + ep := epOnNode("10.0.0.1", "node-a") + ep.Namespace, ep.Name = "ns", "managed" + epStore := cache.NewStore(cache.MetaNamespaceKeyFunc) + if err := epStore.Add(ep); err != nil { + t.Fatalf("seeding endpoint store: %v", err) + } + + ctrl := &ServicesController{Proxy: &recordingProxy{}, svcStore: svcStore, epStore: epStore} + // Deliberately empty: this is the map the callbacks had not filled yet. + ctrl.Services = NewServiceMap() + + got := ctrl.snapshotSource() + if len(got) != 1 { + t.Fatalf("snapshot must hold only the managed service with a valid IP and endpoint, got %d: %v", len(got), got) + } + se, ok := got["ns/managed"] + if !ok || se.Service.Name != "managed" || se.Endpoint.Subsets[0].Addresses[0].IP != "10.0.0.1" { + t.Errorf("unexpected snapshot entry: %+v", got) + } +} + +// Without informers the controller still has to work, which is how the unit +// tests drive it. +func TestSnapshotSourceFallsBackToServiceMap(t *testing.T) { + svc := lbService("192.0.2.10", nil) + svc.Namespace, svc.Name = "ns", "svc" + + ctrl := &ServicesController{Proxy: &recordingProxy{}} + ctrl.Services = NewServiceMap() + ctrl.Services.Set("ns", "svc", &ServiceEndpoints{Service: svc, Endpoint: epOnNode("10.0.0.1", "node-a")}) + + if got := ctrl.snapshotSource(); len(got) != 1 { + t.Errorf("fallback must read the service map, got %v", got) + } +} From 494dbf7b322dcc7280c97752da3b4ba289511c22 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 15 Sep 2026 16:35:53 +0200 Subject: [PATCH 10/42] controllers: requeue the startup cleanup when its retry fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit takePending clears the cleanup flag before the retry runs, and the error branch only logged — so the reconciliation was attempted exactly once and then forgotten, while the message claimed it would try again. The per-service and withdrawal paths do not have this gap: they re-queue themselves from inside the call that failed. Put the flag back, and say what actually happens. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5 (1M context) --- pkg/controllers/services_controller.go | 7 ++++++- pkg/controllers/services_controller_test.go | 23 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/services_controller.go b/pkg/controllers/services_controller.go index 31a218b..baccb3c 100644 --- a/pkg/controllers/services_controller.go +++ b/pkg/controllers/services_controller.go @@ -276,7 +276,12 @@ func (c *ServicesController) retryPending() { if cleanup { log.Info("retrying startup cleanup") if err := c.cleanupRemovedServices(); err != nil { - log.Error(err, "cleanup retry failed, will try again") + // takePending cleared the flag, and unlike the per-service and + // withdrawal paths — which re-queue themselves from inside the + // call that failed — nothing here would put it back. Without this + // the reconciliation is retried exactly once. + c.markCleanupPending() + log.Error(err, "cleanup retry failed, queued again") } } } diff --git a/pkg/controllers/services_controller_test.go b/pkg/controllers/services_controller_test.go index 4eaa39f..0b734bd 100644 --- a/pkg/controllers/services_controller_test.go +++ b/pkg/controllers/services_controller_test.go @@ -659,3 +659,26 @@ func TestSnapshotSourceFallsBackToServiceMap(t *testing.T) { t.Errorf("fallback must read the service map, got %v", got) } } + +// takePending clears the cleanup flag, so a retry that fails again has to put +// it back — otherwise the reconciliation is attempted exactly once and the +// stale state it exists to remove stays for good. +func TestFailedCleanupRetryIsQueuedAgain(t *testing.T) { + px := &recordingProxy{failCleanup: true} + ctrl := &ServicesController{Proxy: px, NodeName: "node-a"} + ctrl.Services = NewServiceMap() + ctrl.markCleanupPending() + + ctrl.retryPending() + if _, _, pending := ctrl.takePending(); !pending { + t.Fatal("a cleanup retry that failed must be queued again") + } + + // And it stops being queued once it succeeds. + ctrl.markCleanupPending() + px.failCleanup = false + ctrl.retryPending() + if _, _, pending := ctrl.takePending(); pending { + t.Error("a successful cleanup retry must clear the flag") + } +} From 9f5a54670f543302d4fc3d16f7306377356863aa Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Thu, 1 Oct 2026 18:33:44 +0200 Subject: [PATCH 11/42] chart: inject NODE_NAME into the DaemonSet The binary scopes ingress rules to the node hosting the backend, and learns that node from NODE_NAME, but the chart never set it. Every supported install therefore fell back to programming every service on every node, which leaves the cross-node hang this branch fixes still reachable. Set it from spec.nodeName through the downward API. This used to live in cozystack's vendored copy of the chart (cozystack/cozystack#3610), where the next chart update would have dropped it. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Mattia Eleuteri --- README.md | 7 ++++--- charts/cozy-proxy/templates/daemonset.yaml | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9c8705c..0360a86 100644 --- a/README.md +++ b/README.md @@ -80,9 +80,10 @@ watches every Service, but programs: | `svc_pod` (`ingress_dnat`), `allowed_ports`, `icmp_allowed_pods` | the node hosting the backend pod | A non-hosting node would rewrite the destination before the packet has even left it. The hosting node then records a conntrack tuple of `(client -> podIP)`, while the reply leaves the pod and gets `saddr` rewritten to the service IP by `egress_snat` at priority `raw`, before conntrack. `(svcIP -> client)` matches nothing, so the reply is not `established` and `port_filter` drops it. | | `pod_svc` (`egress_snat`) | every node | An intra-cluster client is source-NATed by the CNI to its own node address, which the overlay knows how to reach directly. The backend's reply is then tunnelled straight to that node and never traverses the hosting node's netfilter hooks, so the client's node is the only place left where the pod IP can still be turned back into the service IP. Without the entry there, the reply arrives with the wrong source and the client answers with a RST — a cross-node connection that hangs while the same-node one works. | -`NODE_NAME` carries the node identity. When it is unset the ingress scope -check is disabled and everything is programmed everywhere, so the binary still -runs under a chart that does not inject it. +`NODE_NAME` carries the node identity; the chart injects it from +`spec.nodeName`. When it is unset the ingress scope check is disabled and +everything is programmed everywhere, so the binary still runs under a +deployment that does not inject it. ## Installation diff --git a/charts/cozy-proxy/templates/daemonset.yaml b/charts/cozy-proxy/templates/daemonset.yaml index df774bf..c06bdfa 100644 --- a/charts/cozy-proxy/templates/daemonset.yaml +++ b/charts/cozy-proxy/templates/daemonset.yaml @@ -21,6 +21,14 @@ spec: - name: cozy-proxy image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + # Scopes the ingress rules to the backends hosted on this node; see + # "Rule scoping" in the README for why a non-hosting node must not + # program them. + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName securityContext: privileged: true capabilities: From 4b264c4fd39007639da6f820342d902513a51944 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:03:43 +0200 Subject: [PATCH 12/42] docs: propose an L4 LoadBalancer mode Tenants are isolated on egress, and Cilium translates the public IP of a LoadBalancer service inside the client pod, before that egress verdict. A pod of one tenant therefore cannot reach another tenant's Postgres, MariaDB, Ingress or Kubernetes ingress by its public IP, while the Internet can. VM services do not have the problem because cozy-proxy owns them, but its VM mode is a stateless 1:1 NAT that cannot serve services with port translation, several backends or EndpointSlices. The RFC proposes a second datapath, in its own table: a stateful DNAT on the MetalLB announcer towards local ready backends, a masquerade of node sources only, and a guard that drops undeclared traffic so the announcer does not loop it back to its gateway. It records why the two other models tried on the lab fail, how Cilium is told to let go of the VIP, the lifecycle, the security consequences, the chart changes on the Cozystack side, and a rollout and rollback plan per service. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/rfc/l4-loadbalancer-mode.md | 464 +++++++++++++++++++++++++++++++ 1 file changed, 464 insertions(+) create mode 100644 docs/rfc/l4-loadbalancer-mode.md diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md new file mode 100644 index 0000000..0f1113c --- /dev/null +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -0,0 +1,464 @@ +# RFC: L4 LoadBalancer mode + +| | | +|---|---| +| Status | Draft, prototype on branch `feat/l4-loadbalancer` | +| Scope | cozy-proxy, plus the Cozystack charts that would opt services in | +| Supersedes | nothing; the existing VM mode is unchanged | + +## 1. Problem + +Cozystack isolates tenants on **egress**: every tenant namespace carries a +`CiliumClusterwideNetworkPolicy` that only lets its pods talk to their own +tenant, a few platform services, and `world`. Ingress policies are wide open to +`world` and `cluster`. + +That model has a blind spot for public IPs. When a pod connects to the public +IP of a `LoadBalancer` service handled by Cilium (kube-proxy replacement), Cilium +translates the destination **inside the client pod**, before the egress policy is +evaluated. The egress verdict is then taken against the backend's identity and +the post-translation port. A pod of tenant A connecting to the public IP of +tenant B's Postgres is therefore judged as "A talking to B's pod" and dropped, +although the same connection from the Internet succeeds. + +So a tenant cannot reach another tenant's public endpoint (or even a parent +tenant's ingress, for nested projects) by its public IP, which users do +expect: a public IP is public. VM services already behave correctly, because +they are handled by cozy-proxy, which Cilium ignores: the packet leaves the +client pod with the public IP as destination, is evaluated as `world`, leaves the +node and comes back through the normal ingress path. + +The other `LoadBalancer` services cannot use cozy-proxy today: + +- the ingress of tenant Kubernetes clusters, created on the infra side by the + kubevirt cloud-controller-manager (CCM); +- tenant `Ingress` applications (ingress-nginx); +- `Postgres` and `MariaDB` with `external: true`. + +cozy-proxy's VM mode is a stateless 1:1 NAT: it rewrites addresses without +touching ports, assumes exactly one backend, and only reads `v1.Endpoints`. +These services need port translation (CCM services target the tenant node +ports), several backends, EndpointSlices (CCM services are selector-less and +their slices are written by `kubevirt-eps-controller`), readiness, and a stateful +NAT so that several backends can share one IP. + +This RFC proposes a second, independent datapath in cozy-proxy — the **L4 mode** +— that takes such services over from Cilium on the ingress path, so that their +public IP is reachable from every tenant, with the client seen as `world`. + +## 2. Model + +### 2.1 Retained model (E3) + +``` + MetalLB L2 announcer of the VIP (node N) + client ──► VIP:port ──► guard ──► DNAT to a *local* ready backend (conntrack) + └─ if the source is a node IP: masquerade + ◄────────── un-NAT by conntrack on the reply ◄── backend +``` + +1. **DNAT, stateful, on the announcer only.** The node MetalLB elected to answer + ARP for the VIP DNATs new connections to one of its *local* ready backends, + with port translation. Replies come back through the same node's conntrack, + which reverses the translation. Services must use + `externalTrafficPolicy: Local`, which is also what makes MetalLB pick an + announcer that has a local backend. +2. **Masquerade only sources that are node IPs.** An in-cluster pod reaching the + VIP from another node is masqueraded by kube-ovn to *its node's* IP before + leaving that node. If the announcer forwarded that packet unchanged, the + backend would answer the node IP directly, OVN would tunnel the reply + straight to the client's node, and that node would see a reply from the pod IP + to a connection it opened towards the VIP: RST. Masquerading those sources on + the announcer pins the reply to the announcer. Node IPs are read from the + `Node` objects (`InternalIP`), so the rule matches exactly the cluster's + nodes, not a CIDR guess — on some installs the LoadBalancer pool sits in the + node subnet. +3. **Every other source is preserved**: an Internet client is seen with its own + IP, a VM client with its public (VM service) IP. +4. **A guard drops everything else addressed to the VIP.** Without it, a packet + for an undeclared port is not translated, the VIP is not a local address any + more (Cilium released it), and the announcer forwards it to its default + gateway, which sends it straight back: a routing loop until the TTL expires. + This was observed on the lab. + +Observed on the lab with this model (3 nodes, kube-ovn + Cilium chaining, +MetalLB L2): + +| Client | Result | Source seen by the backend | +|---|---|---| +| pod on the announcer | OK | `100.64.0.x` (kube-ovn join IP), Cilium identity `world` | +| pod on another node | OK | `100.64.0.x` of the announcer, identity `world` | +| VM on another node (cozy-proxy VM service) | OK | the VM's public IP | +| Internet | OK | the client's real IP | + +### 2.2 Rejected: DNAT on the announcer without masquerade (E1) + +Works for the Internet and for pods on the announcer. A pod on another node gets +a RST: its node masquerades it to the node IP, the backend answers that node IP, +and OVN carries the SYN-ACK in Geneve directly to the client's node without +going back through the announcer's conntrack. Capture on the client node: + +``` +ovn0 In 10.246.4.189.60638 > VIP.80 [S] +bond0.324 Out 10.200.24.12.45576 > VIP.80 [S] +genev_sys P 10.246.4.186.8080 > 10.200.24.12.45576 [S.] <- straight from the pod +ovn0 Out 10.200.24.12.45576 > 10.246.4.186.8080 [R] +``` + +### 2.3 Rejected: DNAT on every node plus masquerade (E2) + +DNAT on every node, masquerading everything that enters from the pod interface, +fixes pod clients and breaks VM clients. A VM's egress is rewritten by +cozy-proxy's VM mode at priority `raw`, before conntrack (`egress_snat`). With a +DNAT and a masquerade on the VM's own node, the reply is de-masqueraded by +conntrack only after the VM mode's stateless `ingress_dnat` (priority `mangle`) +has already run, so the destination is never turned back into the VM's pod IP, +and the VM mode's `port_filter` drops the packet as a new flow on an +undeclared port. + +### 2.4 How Cilium is told to let go + +The service carries the annotation + +```yaml +service.cilium.io/type: ClusterIP +``` + +Cilium then installs only the ClusterIP frontend: it no longer translates the +VIP (nor the NodePort), while in-cluster traffic to the ClusterIP keeps its +current, isolated behavior. MetalLB still allocates and announces the VIP. +Verified on the lab. + +`service.kubernetes.io/service-proxy-name` is **not** used for this mode: it +makes Cilium drop the ClusterIP and the NodePort as well. + +### 2.5 How cozy-proxy recognizes the service + +A label distinct from the VM mode's: + +```yaml +metadata: + labels: + networking.cozystack.io/lb-proxy: cozy-proxy + annotations: + service.cilium.io/type: ClusterIP +``` + +| Label | Annotation | cozy-proxy L4 mode | +|---|---|---| +| absent | any | ignores the service | +| present | absent or another value | ignores it and logs: Cilium still owns the VIP, two owners would race | +| present | `ClusterIP` | manages the service (if supported, see 2.6) | +| present, and `service-proxy-name: cozy-proxy` | any | ignores it: the VM mode owns the service | + +The label is the trust anchor. The annotation alone is not: the kubevirt CCM +copies the tenant Service's annotations onto the infra Service, so a tenant can +set `service.cilium.io/*` annotations on infra objects, but it cannot set labels +there. + +### 2.6 Supported services, by phase + +| | Phase 1 (MVP) | Later | +|---|---|---| +| Protocol | TCP | UDP | +| `externalTrafficPolicy` | `Local` | `Cluster` (see 6) | +| Address family | IPv4 | IPv6 | +| Ports / backends | many / many | | +| Endpoints | EndpointSlices, ready, local | terminating fallback | +| Announcer source | MetalLB L2 `ServiceL2Status` | BGP, other LB providers | + +A labelled and annotated service that is not supported (for example +`externalTrafficPolicy: Cluster`) is **not** programmed, but its VIP is still +added to the guard, so its traffic is dropped instead of looping. A port the +phase does not support (UDP) is likewise left out of the declared ports, so the +guard drops it. + +## 3. Datapath + +### 3.1 Table + +The L4 mode lives in its own table, `ip cozy_proxy_l4`. It never touches the VM +mode's table `ip cozy_proxy`, and deleting it (`nft delete table ip +cozy_proxy_l4`) removes the L4 mode entirely. + +``` +table ip cozy_proxy_l4 { + # Every VIP of a labelled+annotated service, on every node. + set vips { type ipv4_addr } + # (VIP, protocol, port) of every supported service port, on every node. + set vip_ports { type ipv4_addr . inet_proto . inet_service } + # InternalIP of every Node. + set node_ips { type ipv4_addr } + # One map per announced service port: round-robin slot -> backend. + map backends-//tcp/ { type integer : ipv4_addr . inet_service } + + chain guard { + type filter hook prerouting priority mangle + 10; policy accept; + ct state established,related accept + ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop + } + + chain dnat { + type nat hook prerouting priority dstnat - 5; policy accept; + # announcer only, one rule per (VIP, port) + ip daddr tcp dport dnat ip addr . port to numgen inc mod map @backends-//tcp/ + # announcer with no ready local backend for that port + ip daddr tcp dport drop + } + + chain masq { + type nat hook postrouting priority srcnat - 5; policy accept; + ct status dnat ip saddr @node_ips ct original ip daddr @vips masquerade + } +} +``` + +Notes: + +- `guard` is installed on every node. It accepts `established,related` first, so + ICMP errors about a translated flow (PMTU) still reach the NAT and are + translated with it. New packets to a VIP on an undeclared (VIP, protocol, + port) are dropped, which includes ICMP echo: nobody would answer it anyway. +- `dnat` only holds rules on the announcer. Elsewhere the packet is left alone + and routed out, to the announcer. +- `masq` only matches flows this node translated (`ct status dnat`) towards a + VIP, from a node IP. The masquerade picks the address of the interface towards + the backend, `ovn0` on kube-ovn, hence the `100.64.0.x` source. +- `numgen inc` is per node and per rule, which is enough: only one node + translates a given VIP. + +### 3.2 Priorities, and how they interleave with the VM mode + +| Priority | Hook | Table | Chain | Effect | +|---|---|---|---|---| +| -300 (raw) | prerouting | `cozy_proxy` | `egress_snat` | VM mode: stateless source rewrite pod IP -> VM VIP, before conntrack | +| -200 | prerouting | | conntrack | flow lookup / creation | +| -150 (mangle) | prerouting | `cozy_proxy` | `ingress_dnat` | VM mode: stateless destination rewrite VM VIP -> pod IP | +| -140 | prerouting | `cozy_proxy_l4` | `guard` | L4: drop undeclared traffic to L4 VIPs | +| -105 | prerouting | `cozy_proxy_l4` | `dnat` | L4: stateful DNAT, first packet only | +| -100 | prerouting | | kube-ovn / iptables nat | unchanged | +| 0 (filter) | prerouting | `cozy_proxy` | `port_filter` | VM mode: per-port filter on VM pod IPs | +| 95 | postrouting | `cozy_proxy_l4` | `masq` | L4: masquerade node sources | + +The two modes act on disjoint destination addresses (VM VIPs vs L4 VIPs), so +they do not see each other's packets, with one exception: a VM client of an L4 +service. Its packet leaves its node with the VM's VIP as source (rewritten at +`raw`, before conntrack), is not masqueraded by the announcer (not a node IP), +and the reply comes back to the VM's node as an ordinary packet to the VM VIP. +That is the lab-validated path. + +The opposite case is **not** supported: an L4 backend that is itself the pod of +a VM-mode service. Its replies would be source-rewritten at `raw` before +conntrack, so the reverse translation could not match, and the VM mode's +`port_filter` would see the L4 DNAT as a new flow. The L4 mode excludes such +backends and logs it. + +### 3.3 Programming model + +The L4 table is small (tens of services) and is **rebuilt as a whole** on every +change, in one netlink transaction: + +``` +add table ip cozy_proxy_l4 # so that the delete below cannot fail +delete table ip cozy_proxy_l4 +add table ip cozy_proxy_l4 { ...full desired state... } +``` + +nftables applies a batch atomically: a packet sees either the old or the new +ruleset, never a mix, and conntrack entries are not affected by rule changes. +There is no incremental diff to get wrong, no stale element to garbage-collect, +and a retry is simply the next full sync. This deliberately differs from the VM +mode, whose per-element updates needed the series of fixes on ENOENT handling, +batch isolation and startup snapshots. + +## 4. Lifecycle + +### 4.1 Inputs + +Every instance watches, cluster-wide: + +| Object | Used for | +|---|---| +| `Service` | selection (2.5), VIP (`status.loadBalancer.ingress[].ip`), ports, eTP | +| `EndpointSlice` (`discovery.k8s.io/v1`) | backends: `conditions.ready` (nil counts as ready), not terminating, `nodeName` = this node, port resolved by port name and protocol | +| `Node` | `node_ips` | +| `ServiceL2Status` (`metallb.io/v1beta1`), label `metallb.io/node=` | whether this node announces the service's VIP | + +`NODE_NAME` is mandatory for the L4 mode (the chart already injects it). Without +it the mode refuses to start rather than program every node. + +### 4.2 Reconciliation + +Level-triggered. Every informer event only marks the state dirty. A single sync +loop, rate-limited, recomputes the desired state from the informer caches and +applies it as one transaction (3.3). A failed sync stays dirty and is retried +on the next tick; a periodic resync re-applies the state even without events, so +a table deleted by hand comes back. + +The first sync waits for every informer to be synced. Until then the table left +by the previous instance stays in place and keeps forwarding. + +### 4.3 Conntrack purge + +A conntrack entry outlives the rule that created it. When a backend disappears, +its existing flows would keep being translated to a dead pod until they time out +(hours for TCP), and UDP flows would never recover. + +After every successful sync whose programmed backends changed, and once after +the first sync, the mode deletes the conntrack entries that: + +- were translated by this node (reply source differs from the original + destination), and +- target a VIP the mode knows (previous or current state), and +- point to a (VIP, protocol, port, backend IP, backend port) that is no longer + programmed **on this node** — backend gone or not ready, port removed, + service removed or no longer managed, or this node no longer the announcer. + +Untranslated flows towards a VIP (a pod on a non-announcer node passing through) +are never touched, nor is anything the VM mode tracks. The purge runs after the +commit, so no new flow can be created towards a backend that was just purged. + +### 4.4 Restart and upgrade of the DaemonSet + +- Stopping an instance leaves the table in the kernel. A DaemonSet rolling + update therefore does not interrupt traffic: the old rules keep forwarding until + the new instance's first sync replaces them atomically. Changes that happen + during the gap are applied with that delay, as with kube-proxy. +- Disabling the mode (flag off) deletes the table at startup, which is the + rollback path for the whole mode. + +### 4.5 Announcer changes + +When MetalLB moves the VIP (node drain, backend moved), the new announcer +programs the DNAT on its next sync after its `ServiceL2Status` appears, and the +old one withdraws and purges. Connections in flight are lost, as with any L2 +failover; new connections succeed once both the gratuitous ARP and the sync have +happened. + +## 5. Source address, and what it means for security + +| Client | Source seen by the backend | Cilium identity at the backend | +|---|---|---| +| Internet | its real IP | `world` | +| VM with a public IP (cozy-proxy VM mode) | the VM's public IP | `world` | +| pod, or node process, on any node | `100.64.0.x` of the announcer | `world` | + +Consequences: + +- **Reachability**: a pod of any tenant can reach any L4 service by its public + IP, as from the Internet. That is the goal. ClusterIP traffic is unchanged and + stays isolated. +- **CIDR allowlists now apply to in-cluster clients.** A tenant policy such as + `ingressDeny fromCIDRSet 0.0.0.0/0 except ` is evaluated against + `world` sources. With the VM mode today, an in-cluster client is seen as its + node IP (`remote-node`), which such a rule does not match — it is bypassable. + With the L4 mode the source is `world`, so the deny applies. +- **In-cluster clients are indistinguishable from one another.** Every pod and + node process reaching a VIP through a given announcer shares its `100.64.0.x` + address. An allowlist can admit "the cluster" (the join subnet) or not; it + cannot admit one tenant's pods and not another's. Tenants that need that + distinction must use ClusterIP and network policies, or mTLS. +- **ingress-nginx `whitelist-source-range`, Postgres `pg_hba`** and similar + application-level allowlists see the same addresses as the network policies. +- **Annotation trust**: see 2.5. A tenant copying `service.cilium.io/type` onto + its infra Service through the CCM only makes its own LB dark: without the + label, cozy-proxy does not pick it up. + +## 6. `externalTrafficPolicy` + +| eTP | Behavior | +|---|---| +| `Local` | Phase 1. DNAT to local ready backends on the announcer; Internet and VM sources preserved. | +| `Cluster` | Not programmed (VIP guarded, traffic dropped) until a later phase. Supporting it requires DNAT to backends on other nodes. Their replies would leave through their own node's gateway and miss the announcer's conntrack, so **every** source would have to be masqueraded, losing the client IP — which is precisely what eTP `Cluster` means, but it changes what tenants see today. | + +In production today, all Postgres and Ingress services and most CCM services use +`Local`. MariaDB (`Cluster` by default in the operator) and a minority of tenant +Kubernetes services (the CCM copies the tenant's policy) use `Cluster`. + +## 7. Opting services in, per type + +All of this is on the Cozystack side; cozy-proxy only reacts to the label and +the annotation. Each change sits behind one platform switch so that it can be +enabled per environment. + +| Type | Where | Change | +|---|---|---| +| Postgres | `packages/apps/postgres/templates/external-svc.yaml` | add the label and the annotation; eTP is already `Local` | +| MariaDB | `packages/apps/mariadb/templates/mariadb.yaml` (`spec.service` / `spec.primaryService`) | add the label and the annotation through the operator's service template, and set `externalTrafficPolicy: Local` (operator default is `Cluster`) | +| Tenant Kubernetes (CCM) | `packages/apps/kubernetes/templates/cloud-config.yaml` | label through `infraLabels`, which the CCM already supports. The annotation needs a CCM patch (`infraAnnotations`, next to the existing patches in `packages/apps/kubernetes/images/kubevirt-cloud-provider/patches`). The CCM only sets labels and annotations at creation, so existing infra Services need a one-shot patch. Only tenant Services with eTP `Local` are eligible in phase 1. | +| Ingress | `packages/extra/ingress/templates/nginx-ingress.yaml` | `controller.service.labels` / `annotations`; eTP is already `Local`. The host ingress (`tenant-root`) with PROXY protocol and ouroboros is last, see Open questions. | +| cozy-proxy | `packages/system/cozy-proxy` | bump the vendored chart, enable the mode, extend RBAC (EndpointSlices, Nodes, ServiceL2Status) | + +## 8. Migration and rollback, per service + +Forward, for one service: + +1. Add the label. Nothing happens: cozy-proxy ignores a labelled service without + the annotation. +2. Add the annotation. Cilium withdraws the VIP and cozy-proxy programs it on its + next sync (sub-second). Connections established through Cilium are not + carried over and may be reset. + +Rollback, for one service: **remove the annotation**. Cilium takes the VIP back +and cozy-proxy stops programming it at the same time (it requires both), then +purges its conntrack entries. Removing the label afterwards is cosmetic. + +Removing only the label, on the other hand, leaves the VIP dark (Cilium still +ignores it): the runbook must always remove the annotation. + +Whole mode: remove every annotation, then disable the mode (which deletes the +table). + +## 9. Rollout plan + +Each step runs for at least a week before the next, with the rollback above +rehearsed on the lab first. + +0. **Lab**: a dedicated test service and a cozy-proxy instance with the mode + enabled only on the lab. Matrix: Internet, pod on the announcer, pod + elsewhere, VM client, multiple backends, backend deletion (purge), announcer + failover, DaemonSet restart, ping and undeclared ports (no loop), CIDR deny + policy. +1. **Postgres and MariaDB** (3 services in production): single port, TCP, + single backend, low connection churn, clients mostly known. MariaDB first + needs eTP `Local`. +2. **Tenant Kubernetes, CCM services with eTP `Local`**: port translation and + several backends (worker VMs). Requires the CCM patch. +3. **Ingress**, tenant ingresses first, the host ingress last. This is where the + risk is: 20 Ingress services carry most of the Internet-facing traffic. + +At each step, compare before and after: reachability from the Internet and from +another tenant, the source IP logged by the backend, MetalLB announcer, and the +`cozy_proxy_l4` table on the announcer. + +## 10. Open questions + +1. **Announcer source.** Phase 1 reads MetalLB's `ServiceL2Status`. Upstream may + prefer an abstraction (an interface with a MetalLB L2 implementation), and + MetalLB BGP mode, where every node advertises, would need a different model. +2. **Pre-programming every node that has a local backend** would make failover + hitless and remove the MetalLB dependency. It differs from the rejected E2 + (it does not masquerade the pod interface), but VM clients on a node holding + a local backend would then be translated on their own node. That path has not + been validated on the lab. +3. **Stale ARP after a failover**: a packet for a declared port reaching a node + that no longer announces the VIP is routed back to the gateway until the + caches expire (bounded by the TTL and MetalLB's gratuitous ARP). Should + non-announcers drop VIP traffic that does not come from a local pod + interface? That needs an interface name (`ovn0`), which is CNI specific. +4. **Host-network clients on the announcer** are not translated (no `output` + hook). They reach the VIP only if the gateway hairpins the packet back. Add a + `nat output` rule? +5. **eTP `Cluster`**: support it with full masquerade, keep refusing it, or have + the charts force `Local`? +6. **Terminating endpoints**: fall back to serving-and-terminating backends when + no ready one is left, as kube-proxy does for eTP `Local`? +7. **Node addresses**: only `InternalIP` today. Are there nodes whose pod egress + is masqueraded to another address (multiple uplinks, `ExternalIP`)? +8. **Host ingress with PROXY protocol and ouroboros**: ouroboros handles hairpin + traffic for the PROXY-protocol host ingress. Its interaction with a VIP that + Cilium no longer owns has to be checked before step 3. +9. **Observability**: metrics (sync duration and errors, programmed services, + purged flows) and Events on the Service when it is refused. +10. **Where the label lives**: `networking.cozystack.io/lb-proxy: cozy-proxy` is a + proposal; the maintainers may prefer another key. From 6cdb2bf03f471fe89072438bb3458fc3136b5adf Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:05:31 +0200 Subject: [PATCH 13/42] l4: select the services the L4 mode takes over The L4 mode recognizes its services by the networking.cozystack.io/lb-proxy label, and only manages one that also carries the Cilium annotation service.cilium.io/type: ClusterIP. Without the annotation Cilium keeps translating the VIP, and two owners would race, so the service is ignored. A service the VM mode also selects stays the VM mode's. The label is the trust anchor rather than the annotation: the kubevirt CCM copies a tenant Service's annotations onto the infra Service, so a tenant can set the annotation on an infra object, not the label. A service that is ours but that this phase cannot program (externalTrafficPolicy Cluster) is classified apart: Cilium has released its VIP, so its traffic still has to be guarded. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/l4/select.go | 85 +++++++++++++++++++++++++++++++++++++++++++ pkg/l4/select_test.go | 75 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 pkg/l4/select.go create mode 100644 pkg/l4/select_test.go diff --git a/pkg/l4/select.go b/pkg/l4/select.go new file mode 100644 index 0000000..195cc63 --- /dev/null +++ b/pkg/l4/select.go @@ -0,0 +1,85 @@ +// Package l4 computes the desired state of cozy-proxy's L4 LoadBalancer mode: +// which services it takes over from Cilium, and what this node must program for +// them. It holds no datapath code, so it can be tested anywhere; the nftables +// side lives in pkg/proxy. +// +// See docs/rfc/l4-loadbalancer-mode.md for the model. +package l4 + +import ( + v1 "k8s.io/api/core/v1" +) + +const ( + // ProxyLabel selects a Service for the L4 mode. It is deliberately not the + // VM mode's service.kubernetes.io/service-proxy-name: that label also makes + // Cilium drop the ClusterIP and the NodePort, while the L4 mode only takes + // over the public IP. + ProxyLabel = "networking.cozystack.io/lb-proxy" + ProxyLabelValue = "cozy-proxy" + + // CiliumTypeAnnotation set to CiliumTypeClusterIP makes Cilium install only + // the ClusterIP frontend, releasing the VIP. The L4 mode requires it: without + // it Cilium keeps translating the VIP and the two would race. + CiliumTypeAnnotation = "service.cilium.io/type" + CiliumTypeClusterIP = "ClusterIP" + + vmProxyNameLabel = "service.kubernetes.io/service-proxy-name" + vmProxyName = "cozy-proxy" +) + +// Selection says how the L4 mode treats a Service. +type Selection int + +const ( + // NotSelected services do not carry the label and are none of our business. + NotSelected Selection = iota + // Ignored services carry the label but must not be touched, because some + // other component still owns the VIP. + Ignored + // Unsupported services are ours — Cilium has released the VIP — but this + // version cannot program them. Their VIP is still guarded, so that its + // traffic is dropped instead of being routed back to the gateway. + Unsupported + // Managed services are programmed. + Managed +) + +func (s Selection) String() string { + switch s { + case NotSelected: + return "NotSelected" + case Ignored: + return "Ignored" + case Unsupported: + return "Unsupported" + case Managed: + return "Managed" + } + return "Selection(?)" +} + +// Classify decides how the L4 mode treats svc. The reason explains anything +// other than Managed or NotSelected, and is meant for the log. +// +// The label is the trust anchor, not the annotation: the kubevirt CCM copies a +// tenant Service's annotations onto the infra Service, so a tenant can set the +// annotation on an infra object, but not the label. +func Classify(svc *v1.Service) (Selection, string) { + if svc == nil || svc.Labels[ProxyLabel] != ProxyLabelValue { + return NotSelected, "" + } + if svc.Labels[vmProxyNameLabel] == vmProxyName { + return Ignored, "also selected by the VM mode, which owns it" + } + if svc.Spec.Type != v1.ServiceTypeLoadBalancer { + return Ignored, "not a LoadBalancer service" + } + if svc.Annotations[CiliumTypeAnnotation] != CiliumTypeClusterIP { + return Ignored, "missing " + CiliumTypeAnnotation + ": " + CiliumTypeClusterIP + ", Cilium still owns the VIP" + } + if svc.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyLocal { + return Unsupported, "externalTrafficPolicy must be Local" + } + return Managed, "" +} diff --git a/pkg/l4/select_test.go b/pkg/l4/select_test.go new file mode 100644 index 0000000..876d82a --- /dev/null +++ b/pkg/l4/select_test.go @@ -0,0 +1,75 @@ +package l4 + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// l4Service builds a LoadBalancer service carrying the L4 label and the Cilium +// annotation, with externalTrafficPolicy Local: the shape the L4 mode manages. +func l4Service(ns, name string, vips ...string) *v1.Service { + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: map[string]string{ProxyLabel: ProxyLabelValue}, + Annotations: map[string]string{CiliumTypeAnnotation: CiliumTypeClusterIP}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, + }, + } + for _, ip := range vips { + svc.Status.LoadBalancer.Ingress = append(svc.Status.LoadBalancer.Ingress, v1.LoadBalancerIngress{IP: ip}) + } + return svc +} + +func TestClassify(t *testing.T) { + cases := []struct { + name string + mutate func(*v1.Service) + want Selection + }{ + {"label, annotation, eTP Local", func(*v1.Service) {}, Managed}, + {"no label", func(s *v1.Service) { delete(s.Labels, ProxyLabel) }, NotSelected}, + {"label with another value", func(s *v1.Service) { s.Labels[ProxyLabel] = "kube-router" }, NotSelected}, + // Cilium still translates the VIP: programming it too would give the + // address two owners. + {"annotation missing", func(s *v1.Service) { delete(s.Annotations, CiliumTypeAnnotation) }, Ignored}, + {"annotation with another value", func(s *v1.Service) { s.Annotations[CiliumTypeAnnotation] = "LoadBalancer" }, Ignored}, + // The VM mode owns the service; it must keep its behavior untouched. + {"also selected by the VM mode", func(s *v1.Service) { + s.Labels["service.kubernetes.io/service-proxy-name"] = "cozy-proxy" + }, Ignored}, + {"not a LoadBalancer", func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeClusterIP }, Ignored}, + // Cilium has let go of the VIP, so it must at least be guarded, but + // phase 1 cannot program it. + {"eTP Cluster", func(s *v1.Service) { + s.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyCluster + }, Unsupported}, + {"eTP unset defaults to Cluster", func(s *v1.Service) { s.Spec.ExternalTrafficPolicy = "" }, Unsupported}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + svc := l4Service("ns", "svc", "192.0.2.10") + c.mutate(svc) + got, reason := Classify(svc) + if got != c.want { + t.Errorf("Classify = %v (%s), want %v", got, reason, c.want) + } + if got != Managed && got != NotSelected && reason == "" { + t.Errorf("a %v service must come with a reason", got) + } + }) + } +} + +func TestClassifyNil(t *testing.T) { + if got, _ := Classify(nil); got != NotSelected { + t.Errorf("Classify(nil) = %v, want NotSelected", got) + } +} From 7377375d8b027af506f0044568f0a9c3b4316a35 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:07:47 +0200 Subject: [PATCH 14/42] l4: compute what a node programs for the L4 mode Build turns the cluster state into what one node has to program: the VIPs to guard, the declared ports, the node IPs to masquerade, and the frontends to translate. It is a pure function, so the whole selection logic is tested without a kernel. Every node guards every VIP and lets the declared ports through, so a pod anywhere reaches the announcer. Only the node MetalLB announces the VIP from translates it, towards its local ready backends: that is the model validated on the lab, where translating on the other nodes broke either pod or VM clients. Backends come from EndpointSlices, which covers the selector-less services the kubevirt CCM creates, with the port resolved by name and protocol as kube-proxy does. A nil Ready counts as ready, as the API says. An announced port with no local backend is kept as an empty rule, to be dropped rather than routed back to the gateway. Node IPs are read from the Node objects rather than guessed as a CIDR: the LoadBalancer pool can sit inside the node subnet, and a VM client's public IP must keep its source. The pods of VM-mode services are excluded as backends: their replies are source-rewritten before conntrack, so a translation towards them could not be reversed. UDP is left out for now and reported. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/l4/state.go | 293 ++++++++++++++++++++++++++++++ pkg/l4/state_test.go | 416 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 709 insertions(+) create mode 100644 pkg/l4/state.go create mode 100644 pkg/l4/state_test.go diff --git a/pkg/l4/state.go b/pkg/l4/state.go new file mode 100644 index 0000000..e4a17b7 --- /dev/null +++ b/pkg/l4/state.go @@ -0,0 +1,293 @@ +package l4 + +import ( + "cmp" + "fmt" + "net/netip" + "slices" + "strings" + + v1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" +) + +// Backend is one translation target: a ready endpoint hosted on this node, at +// the port the EndpointSlice resolved for the service port. +type Backend struct { + IP netip.Addr + Port uint16 +} + +func (b Backend) String() string { return netip.AddrPortFrom(b.IP, b.Port).String() } + +// PortKey is a declared (VIP, protocol, port) frontend. +type PortKey struct { + VIP netip.Addr + Protocol v1.Protocol + Port uint16 +} + +// Rule is a frontend this node translates. An empty Backends means the port is +// announced here but has no local ready backend: its traffic is dropped, since +// leaving it untranslated would route it back to the gateway. +type Rule struct { + PortKey + // Service is the namespace/name of the Service the frontend belongs to. + Service string + Backends []Backend +} + +// State is everything one node programs for the L4 mode. Every slice is sorted, +// so two states can be compared with reflect.DeepEqual. +type State struct { + // VIPs are guarded on every node: new traffic to them that is not on a + // declared port is dropped. + VIPs []netip.Addr + // Ports are the declared frontends of every supported service, on every + // node, including the ones this node does not announce: a pod here must be + // able to reach them through the announcer. + Ports []PortKey + // NodeIPs are the sources masqueraded after translation. + NodeIPs []netip.Addr + // Rules are the frontends this node announces and therefore translates. + Rules []Rule +} + +// Notice explains why a service, or part of it, is not programmed. +type Notice struct { + Service string + Message string +} + +// Input is the cluster state Build reads. +type Input struct { + // NodeName is the node this instance programs. + NodeName string + // Services and EndpointSlices hold every object in the cluster, not only + // the L4 ones: the VM mode's slices are needed to exclude its pods. + Services []*v1.Service + EndpointSlices []*discoveryv1.EndpointSlice + Nodes []*v1.Node + // Announced holds the namespace/name of the services whose VIP this node + // announces. + Announced map[string]bool +} + +// Build computes the state this node must program, and the notices worth +// logging. It is a pure function of its input. +func Build(in Input) (State, []Notice) { + b := builder{in: in, slices: groupSlices(in.EndpointSlices), owner: map[PortKey]string{}} + b.vmPods = b.vmModePods() + + services := slices.Clone(in.Services) + slices.SortFunc(services, func(x, y *v1.Service) int { return cmp.Compare(serviceKey(x), serviceKey(y)) }) + for _, svc := range services { + b.addService(svc) + } + + b.st.NodeIPs = nodeIPs(in.Nodes) + b.st.VIPs = sortedUnique(b.st.VIPs, netip.Addr.Compare) + slices.SortFunc(b.st.Ports, comparePortKey) + slices.SortFunc(b.st.Rules, func(x, y Rule) int { return comparePortKey(x.PortKey, y.PortKey) }) + return b.st, b.notices +} + +type builder struct { + in Input + slices map[string][]*discoveryv1.EndpointSlice + vmPods map[netip.Addr]string + owner map[PortKey]string + st State + notices []Notice +} + +func (b *builder) notice(service, format string, args ...any) { + b.notices = append(b.notices, Notice{Service: service, Message: fmt.Sprintf(format, args...)}) +} + +func (b *builder) addService(svc *v1.Service) { + key := serviceKey(svc) + sel, reason := Classify(svc) + switch sel { + case NotSelected: + return + case Ignored: + b.notice(key, "ignored: %s", reason) + return + } + + vips := serviceVIPs(svc) + b.st.VIPs = append(b.st.VIPs, vips...) + if sel == Unsupported { + b.notice(key, "not programmed, its VIP drops all traffic: %s", reason) + return + } + + announced := b.in.Announced[key] + for _, sp := range svc.Spec.Ports { + proto := sp.Protocol + if proto == "" { + proto = v1.ProtocolTCP + } + if proto != v1.ProtocolTCP { + b.notice(key, "port %s/%d not programmed: only TCP is supported", proto, sp.Port) + continue + } + var backends []Backend + if announced { + backends = b.localBackends(key, svc.Namespace, svc.Name, sp.Name, proto) + } + for _, vip := range vips { + pk := PortKey{VIP: vip, Protocol: proto, Port: uint16(sp.Port)} + if prev, taken := b.owner[pk]; taken { + b.notice(key, "port %s %s/%d not programmed: already served by %s", vip, proto, sp.Port, prev) + continue + } + b.owner[pk] = key + b.st.Ports = append(b.st.Ports, pk) + if announced { + b.st.Rules = append(b.st.Rules, Rule{PortKey: pk, Service: key, Backends: backends}) + } + } + } +} + +// localBackends returns the ready endpoints of the service port hosted on this +// node, sorted and deduplicated. The port is matched by name and protocol, the +// way kube-proxy does, which also resolves named target ports. +func (b *builder) localBackends(key, ns, name, portName string, proto v1.Protocol) []Backend { + var out []Backend + for _, s := range b.slices[ns+"/"+name] { + if s.AddressType != discoveryv1.AddressTypeIPv4 { + continue + } + port, ok := slicePort(s, portName, proto) + if !ok { + continue + } + for _, ep := range s.Endpoints { + if !ready(ep) || ep.NodeName == nil || *ep.NodeName != b.in.NodeName || len(ep.Addresses) == 0 { + continue + } + ip, err := netip.ParseAddr(ep.Addresses[0]) + if err != nil || !ip.Is4() { + continue + } + if owner, vm := b.vmPods[ip]; vm { + b.notice(key, "backend %s excluded: it is the pod of VM-mode service %s", ip, owner) + continue + } + out = append(out, Backend{IP: ip, Port: port}) + } + } + return sortedUnique(out, func(x, y Backend) int { + return cmp.Or(x.IP.Compare(y.IP), cmp.Compare(x.Port, y.Port)) + }) +} + +// vmModePods returns the pod IPs backing a VM-mode service. The VM mode +// rewrites their replies' source at priority raw, before conntrack, so an L4 +// translation towards them could never be reversed. +func (b *builder) vmModePods() map[netip.Addr]string { + out := map[netip.Addr]string{} + for _, svc := range b.in.Services { + if svc == nil || svc.Labels[vmProxyNameLabel] != vmProxyName { + continue + } + for _, s := range b.slices[serviceKey(svc)] { + for _, ep := range s.Endpoints { + for _, a := range ep.Addresses { + if ip, err := netip.ParseAddr(a); err == nil { + out[ip] = serviceKey(svc) + } + } + } + } + } + return out +} + +// ready follows the EndpointSlice API: a nil Ready is to be read as ready. +// A terminating endpoint is not ready. +func ready(ep discoveryv1.Endpoint) bool { + return ep.Conditions.Ready == nil || *ep.Conditions.Ready +} + +func slicePort(s *discoveryv1.EndpointSlice, name string, proto v1.Protocol) (uint16, bool) { + for _, p := range s.Ports { + pName := "" + if p.Name != nil { + pName = *p.Name + } + pProto := v1.ProtocolTCP + if p.Protocol != nil { + pProto = *p.Protocol + } + if pName == name && pProto == proto && p.Port != nil && *p.Port > 0 && *p.Port <= 65535 { + return uint16(*p.Port), true + } + } + return 0, false +} + +func serviceVIPs(svc *v1.Service) []netip.Addr { + var out []netip.Addr + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ip, err := netip.ParseAddr(ing.IP); err == nil && ip.Is4() { + out = append(out, ip) + } + } + return out +} + +func nodeIPs(nodes []*v1.Node) []netip.Addr { + var out []netip.Addr + for _, n := range nodes { + if n == nil { + continue + } + for _, a := range n.Status.Addresses { + if a.Type != v1.NodeInternalIP { + continue + } + if ip, err := netip.ParseAddr(a.Address); err == nil && ip.Is4() { + out = append(out, ip) + } + } + } + return sortedUnique(out, netip.Addr.Compare) +} + +func groupSlices(all []*discoveryv1.EndpointSlice) map[string][]*discoveryv1.EndpointSlice { + out := map[string][]*discoveryv1.EndpointSlice{} + for _, s := range all { + if s == nil { + continue + } + svc := s.Labels[discoveryv1.LabelServiceName] + if svc == "" { + continue + } + k := s.Namespace + "/" + svc + out[k] = append(out[k], s) + } + return out +} + +func serviceKey(svc *v1.Service) string { return svc.Namespace + "/" + svc.Name } + +func comparePortKey(x, y PortKey) int { + return cmp.Or( + x.VIP.Compare(y.VIP), + strings.Compare(string(x.Protocol), string(y.Protocol)), + cmp.Compare(x.Port, y.Port), + ) +} + +func sortedUnique[T comparable](s []T, compare func(T, T) int) []T { + if len(s) == 0 { + return nil + } + slices.SortFunc(s, compare) + return slices.Compact(s) +} diff --git a/pkg/l4/state_test.go b/pkg/l4/state_test.go new file mode 100644 index 0000000..d667e8b --- /dev/null +++ b/pkg/l4/state_test.go @@ -0,0 +1,416 @@ +package l4 + +import ( + "net/netip" + "reflect" + "strings" + "testing" + + v1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const thisNode = "node-a" + +func ptr[T any](v T) *T { return &v } + +// endpoint is a ready endpoint on the given node. +func endpoint(ip, node string) discoveryv1.Endpoint { + return discoveryv1.Endpoint{ + Addresses: []string{ip}, + Conditions: discoveryv1.EndpointConditions{Ready: ptr(true)}, + NodeName: ptr(node), + } +} + +// slice builds an IPv4 EndpointSlice for the service ns/svc. +func slice(ns, svc, name string, ports []discoveryv1.EndpointPort, eps ...discoveryv1.Endpoint) *discoveryv1.EndpointSlice { + return &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: map[string]string{discoveryv1.LabelServiceName: svc}, + }, + AddressType: discoveryv1.AddressTypeIPv4, + Ports: ports, + Endpoints: eps, + } +} + +func epPort(name string, port int32) discoveryv1.EndpointPort { + return discoveryv1.EndpointPort{Name: ptr(name), Port: ptr(port), Protocol: ptr(v1.ProtocolTCP)} +} + +func svcPort(name string, port int32) v1.ServicePort { + return v1.ServicePort{Name: name, Port: port, Protocol: v1.ProtocolTCP} +} + +func node(name string, addrs ...v1.NodeAddress) *v1.Node { + return &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: v1.NodeStatus{Addresses: addrs}, + } +} + +func internalIP(ip string) v1.NodeAddress { + return v1.NodeAddress{Type: v1.NodeInternalIP, Address: ip} +} + +func addr(s string) netip.Addr { return netip.MustParseAddr(s) } + +func be(ip string, port uint16) Backend { return Backend{IP: addr(ip), Port: port} } + +func key(vip string, port uint16) PortKey { + return PortKey{VIP: addr(vip), Protocol: v1.ProtocolTCP, Port: port} +} + +// The phase 1 shape: one announced service, several ports, only the ready +// backends hosted here. +func TestBuildAnnouncedServiceUsesLocalReadyBackends(t *testing.T) { + svc := l4Service("ns", "pg", "192.0.2.10") + svc.Spec.Ports = []v1.ServicePort{svcPort("http", 80), svcPort("https", 443)} + + notReady := endpoint("10.0.0.4", thisNode) + notReady.Conditions.Ready = ptr(false) + terminating := endpoint("10.0.0.5", thisNode) + terminating.Conditions.Terminating = ptr(true) + terminating.Conditions.Ready = ptr(false) + unknown := endpoint("10.0.0.6", thisNode) + unknown.Conditions.Ready = nil // nil means ready, per the API + + in := Input{ + NodeName: thisNode, + Services: []*v1.Service{svc}, + EndpointSlices: []*discoveryv1.EndpointSlice{slice("ns", "pg", "pg-1", + // Target ports are already resolved in the slice, named targets + // included; they need not match the service port. + []discoveryv1.EndpointPort{epPort("http", 8080), epPort("https", 8443)}, + endpoint("10.0.0.2", thisNode), + endpoint("10.0.0.1", thisNode), + endpoint("10.0.0.3", "node-b"), // remote: eTP Local only uses local backends + notReady, + terminating, + unknown, + )}, + Announced: map[string]bool{"ns/pg": true}, + } + + got, notices := Build(in) + + want := State{ + VIPs: []netip.Addr{addr("192.0.2.10")}, + Ports: []PortKey{key("192.0.2.10", 80), key("192.0.2.10", 443)}, + Rules: []Rule{ + {PortKey: key("192.0.2.10", 80), Service: "ns/pg", + Backends: []Backend{be("10.0.0.1", 8080), be("10.0.0.2", 8080), be("10.0.0.6", 8080)}}, + {PortKey: key("192.0.2.10", 443), Service: "ns/pg", + Backends: []Backend{be("10.0.0.1", 8443), be("10.0.0.2", 8443), be("10.0.0.6", 8443)}}, + }, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Build =\n%+v\nwant\n%+v", got, want) + } + if len(notices) != 0 { + t.Errorf("no notice expected, got %v", notices) + } +} + +// Only the announcer translates. Every other node still guards the VIP, and +// lets declared ports through so that its pods' packets reach the announcer. +func TestBuildNotAnnouncedServiceIsGuardedOnly(t *testing.T) { + svc := l4Service("ns", "pg", "192.0.2.10") + svc.Spec.Ports = []v1.ServicePort{svcPort("pg", 5432)} + + got, _ := Build(Input{ + NodeName: thisNode, + Services: []*v1.Service{svc}, + EndpointSlices: []*discoveryv1.EndpointSlice{slice("ns", "pg", "pg-1", + []discoveryv1.EndpointPort{epPort("pg", 5432)}, endpoint("10.0.0.1", thisNode))}, + }) + + if len(got.Rules) != 0 { + t.Errorf("a node that does not announce the VIP must not translate it, got %+v", got.Rules) + } + if !reflect.DeepEqual(got.VIPs, []netip.Addr{addr("192.0.2.10")}) { + t.Errorf("VIPs = %v, want the service VIP", got.VIPs) + } + if !reflect.DeepEqual(got.Ports, []PortKey{key("192.0.2.10", 5432)}) { + t.Errorf("Ports = %v, want the declared port", got.Ports) + } +} + +// An announced port without any local ready backend must be dropped, not left +// untranslated: an untranslated packet would be routed back to the gateway. +func TestBuildPortWithoutBackendIsAnEmptyRule(t *testing.T) { + svc := l4Service("ns", "pg", "192.0.2.10") + svc.Spec.Ports = []v1.ServicePort{svcPort("pg", 5432), svcPort("metrics", 9187)} + + got, _ := Build(Input{ + NodeName: thisNode, + Services: []*v1.Service{svc}, + EndpointSlices: []*discoveryv1.EndpointSlice{slice("ns", "pg", "pg-1", + []discoveryv1.EndpointPort{epPort("pg", 5432)}, // no "metrics" port in the slice + endpoint("10.0.0.1", "node-b"))}, // and no local backend at all + Announced: map[string]bool{"ns/pg": true}, + }) + + want := []Rule{ + {PortKey: key("192.0.2.10", 5432), Service: "ns/pg"}, + {PortKey: key("192.0.2.10", 9187), Service: "ns/pg"}, + } + if !reflect.DeepEqual(got.Rules, want) { + t.Errorf("Rules = %+v, want empty rules for both ports %+v", got.Rules, want) + } +} + +// The kubevirt CCM creates selector-less services whose single port has no +// name, and writes their slices itself. +func TestBuildSelectorlessCCMService(t *testing.T) { + svc := l4Service("tenant-k8s", "a6ed08", "192.0.2.20") + svc.Spec.Ports = []v1.ServicePort{{Port: 80, Protocol: v1.ProtocolTCP}} + + s := slice("tenant-k8s", "a6ed08", "a6ed08-x", []discoveryv1.EndpointPort{ + {Name: ptr(""), Port: ptr(int32(31790)), Protocol: ptr(v1.ProtocolTCP)}, + }, endpoint("10.0.80.67", thisNode)) + s.Labels[discoveryv1.LabelManagedBy] = "kubevirt-eps-controller" + + got, _ := Build(Input{ + NodeName: thisNode, + Services: []*v1.Service{svc}, + EndpointSlices: []*discoveryv1.EndpointSlice{s}, + Announced: map[string]bool{"tenant-k8s/a6ed08": true}, + }) + + want := []Rule{{PortKey: key("192.0.2.20", 80), Service: "tenant-k8s/a6ed08", + Backends: []Backend{be("10.0.80.67", 31790)}}} + if !reflect.DeepEqual(got.Rules, want) { + t.Errorf("Rules = %+v, want %+v", got.Rules, want) + } +} + +// Endpoints of one service can transiently appear in two slices. +func TestBuildDeduplicatesBackendsAcrossSlices(t *testing.T) { + svc := l4Service("ns", "web", "192.0.2.10") + svc.Spec.Ports = []v1.ServicePort{svcPort("http", 80)} + ports := []discoveryv1.EndpointPort{epPort("http", 8080)} + + got, _ := Build(Input{ + NodeName: thisNode, + Services: []*v1.Service{svc}, + EndpointSlices: []*discoveryv1.EndpointSlice{ + slice("ns", "web", "web-1", ports, endpoint("10.0.0.1", thisNode)), + slice("ns", "web", "web-2", ports, endpoint("10.0.0.1", thisNode), endpoint("10.0.0.2", thisNode)), + // Same name, other namespace: not this service's slice. + slice("other", "web", "web-3", ports, endpoint("10.0.0.9", thisNode)), + }, + Announced: map[string]bool{"ns/web": true}, + }) + + want := []Backend{be("10.0.0.1", 8080), be("10.0.0.2", 8080)} + if len(got.Rules) != 1 || !reflect.DeepEqual(got.Rules[0].Backends, want) { + t.Errorf("Rules = %+v, want one rule with %v", got.Rules, want) + } +} + +func TestBuildIgnoresNonIPv4AndUnscheduledEndpoints(t *testing.T) { + svc := l4Service("ns", "web", "192.0.2.10", "2001:db8::10") + svc.Spec.Ports = []v1.ServicePort{svcPort("http", 80)} + ports := []discoveryv1.EndpointPort{epPort("http", 8080)} + + v6 := slice("ns", "web", "web-v6", ports, endpoint("2001:db8::1", thisNode)) + v6.AddressType = discoveryv1.AddressTypeIPv6 + noNode := endpoint("10.0.0.2", thisNode) + noNode.NodeName = nil + + got, _ := Build(Input{ + NodeName: thisNode, + Services: []*v1.Service{svc}, + EndpointSlices: []*discoveryv1.EndpointSlice{v6, slice("ns", "web", "web-v4", ports, endpoint("10.0.0.1", thisNode), noNode)}, + Announced: map[string]bool{"ns/web": true}, + }) + + if !reflect.DeepEqual(got.VIPs, []netip.Addr{addr("192.0.2.10")}) { + t.Errorf("VIPs = %v, want only the IPv4 one", got.VIPs) + } + want := []Rule{{PortKey: key("192.0.2.10", 80), Service: "ns/web", Backends: []Backend{be("10.0.0.1", 8080)}}} + if !reflect.DeepEqual(got.Rules, want) { + t.Errorf("Rules = %+v, want %+v", got.Rules, want) + } +} + +// Phase 1 is TCP only. A UDP port is left out of the declared ports, so the +// guard drops it rather than letting it loop. +func TestBuildLeavesUDPPortsOut(t *testing.T) { + svc := l4Service("ns", "dns", "192.0.2.10") + svc.Spec.Ports = []v1.ServicePort{svcPort("tcp", 53), {Name: "udp", Port: 53, Protocol: v1.ProtocolUDP}} + + got, notices := Build(Input{ + NodeName: thisNode, + Services: []*v1.Service{svc}, + EndpointSlices: []*discoveryv1.EndpointSlice{slice("ns", "dns", "dns-1", []discoveryv1.EndpointPort{ + epPort("tcp", 53), + {Name: ptr("udp"), Port: ptr(int32(53)), Protocol: ptr(v1.ProtocolUDP)}, + }, endpoint("10.0.0.1", thisNode))}, + Announced: map[string]bool{"ns/dns": true}, + }) + + if !reflect.DeepEqual(got.Ports, []PortKey{key("192.0.2.10", 53)}) { + t.Errorf("Ports = %v, want only TCP/53", got.Ports) + } + if len(got.Rules) != 1 || got.Rules[0].Protocol != v1.ProtocolTCP { + t.Errorf("Rules = %+v, want only the TCP rule", got.Rules) + } + if !hasNotice(notices, "ns/dns", "UDP") { + t.Errorf("a skipped port must be reported, got %v", notices) + } +} + +// A service Cilium released but phase 1 cannot program is guarded: its VIP +// drops everything. +func TestBuildUnsupportedServiceIsGuardedWithNoPort(t *testing.T) { + svc := l4Service("ns", "maria", "192.0.2.10") + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyCluster + svc.Spec.Ports = []v1.ServicePort{svcPort("mysql", 3306)} + + got, notices := Build(Input{ + NodeName: thisNode, + Services: []*v1.Service{svc}, + Announced: map[string]bool{"ns/maria": true}, + }) + + if !reflect.DeepEqual(got.VIPs, []netip.Addr{addr("192.0.2.10")}) { + t.Errorf("VIPs = %v, want the VIP guarded", got.VIPs) + } + if len(got.Ports) != 0 || len(got.Rules) != 0 { + t.Errorf("an unsupported service must not be opened, got ports %v rules %v", got.Ports, got.Rules) + } + if !hasNotice(notices, "ns/maria", "externalTrafficPolicy") { + t.Errorf("an unsupported service must be reported, got %v", notices) + } +} + +// Services some other component owns are left entirely alone, guard included. +func TestBuildIgnoredAndUnselectedServicesAreUntouched(t *testing.T) { + noAnnotation := l4Service("ns", "half", "192.0.2.10") + delete(noAnnotation.Annotations, CiliumTypeAnnotation) + plain := l4Service("ns", "plain", "192.0.2.11") + delete(plain.Labels, ProxyLabel) + + got, notices := Build(Input{NodeName: thisNode, Services: []*v1.Service{noAnnotation, plain}}) + + if len(got.VIPs) != 0 || len(got.Ports) != 0 || len(got.Rules) != 0 { + t.Errorf("nothing must be programmed, got %+v", got) + } + if !hasNotice(notices, "ns/half", CiliumTypeAnnotation) { + t.Errorf("an ignored labelled service must be reported, got %v", notices) + } + if hasNotice(notices, "ns/plain", "") { + t.Errorf("an unselected service is none of our business, got %v", notices) + } +} + +// A service waiting for its VIP contributes nothing yet. +func TestBuildServiceWithoutVIP(t *testing.T) { + svc := l4Service("ns", "pending") + svc.Spec.Ports = []v1.ServicePort{svcPort("pg", 5432)} + got, _ := Build(Input{NodeName: thisNode, Services: []*v1.Service{svc}, Announced: map[string]bool{"ns/pending": true}}) + if len(got.VIPs) != 0 || len(got.Ports) != 0 || len(got.Rules) != 0 { + t.Errorf("nothing must be programmed before the VIP is allocated, got %+v", got) + } +} + +// Masquerading is restricted to the nodes' own addresses, read from the Node +// objects rather than guessed as a CIDR: the LoadBalancer pool can sit in the +// node subnet, and a VM client's public IP must not be masqueraded. +func TestBuildNodeIPs(t *testing.T) { + got, _ := Build(Input{ + NodeName: thisNode, + Nodes: []*v1.Node{ + node("node-b", internalIP("10.200.24.12"), v1.NodeAddress{Type: v1.NodeHostName, Address: "node-b"}), + node("node-a", internalIP("10.200.24.11"), internalIP("fd00::11"), + v1.NodeAddress{Type: v1.NodeExternalIP, Address: "198.51.100.1"}), + node("node-c", internalIP("10.200.24.12")), // duplicate address + }, + }) + want := []netip.Addr{addr("10.200.24.11"), addr("10.200.24.12")} + if !reflect.DeepEqual(got.NodeIPs, want) { + t.Errorf("NodeIPs = %v, want %v", got.NodeIPs, want) + } +} + +// A VM-mode pod rewrites its replies' source at priority raw, before +// conntrack, so an L4 translation towards it could never be reversed. +func TestBuildExcludesBackendsOwnedByTheVMMode(t *testing.T) { + vm := l4Service("tenant", "vm", "192.0.2.50") + delete(vm.Annotations, CiliumTypeAnnotation) + delete(vm.Labels, ProxyLabel) + vm.Labels[vmProxyNameLabel] = vmProxyName + + web := l4Service("tenant", "web", "192.0.2.10") + web.Spec.Ports = []v1.ServicePort{svcPort("http", 80)} + ports := []discoveryv1.EndpointPort{epPort("http", 8080)} + + got, notices := Build(Input{ + NodeName: thisNode, + Services: []*v1.Service{vm, web}, + EndpointSlices: []*discoveryv1.EndpointSlice{ + slice("tenant", "vm", "vm-1", nil, endpoint("10.0.0.7", thisNode)), + slice("tenant", "web", "web-1", ports, endpoint("10.0.0.7", thisNode), endpoint("10.0.0.8", thisNode)), + }, + Announced: map[string]bool{"tenant/web": true}, + }) + + want := []Backend{be("10.0.0.8", 8080)} + if len(got.Rules) != 1 || !reflect.DeepEqual(got.Rules[0].Backends, want) { + t.Errorf("Rules = %+v, want only %v", got.Rules, want) + } + if !hasNotice(notices, "tenant/web", "10.0.0.7") { + t.Errorf("an excluded backend must be reported, got %v", notices) + } +} + +// MetalLB can share a VIP between services on distinct ports. Two services +// declaring the same port cannot both be served; the first one in key order +// keeps it. +func TestBuildSharedVIP(t *testing.T) { + a := l4Service("ns", "a", "192.0.2.10") + a.Spec.Ports = []v1.ServicePort{svcPort("http", 80)} + b := l4Service("ns", "b", "192.0.2.10") + b.Spec.Ports = []v1.ServicePort{svcPort("http", 80), svcPort("ssh", 22)} + ports := []discoveryv1.EndpointPort{epPort("http", 8080), epPort("ssh", 22)} + + got, notices := Build(Input{ + NodeName: thisNode, + // Deliberately out of order: the result must not depend on it. + Services: []*v1.Service{b, a}, + EndpointSlices: []*discoveryv1.EndpointSlice{ + slice("ns", "a", "a-1", ports, endpoint("10.0.0.1", thisNode)), + slice("ns", "b", "b-1", ports, endpoint("10.0.0.2", thisNode)), + }, + Announced: map[string]bool{"ns/a": true, "ns/b": true}, + }) + + if !reflect.DeepEqual(got.VIPs, []netip.Addr{addr("192.0.2.10")}) { + t.Errorf("VIPs = %v, want the shared VIP once", got.VIPs) + } + want := []Rule{ + {PortKey: key("192.0.2.10", 22), Service: "ns/b", Backends: []Backend{be("10.0.0.2", 22)}}, + {PortKey: key("192.0.2.10", 80), Service: "ns/a", Backends: []Backend{be("10.0.0.1", 8080)}}, + } + if !reflect.DeepEqual(got.Rules, want) { + t.Errorf("Rules = %+v, want %+v", got.Rules, want) + } + if !hasNotice(notices, "ns/b", "ns/a") { + t.Errorf("the conflict must be reported on the losing service, got %v", notices) + } +} + +func hasNotice(notices []Notice, service, substr string) bool { + for _, n := range notices { + if n.Service == service && strings.Contains(n.Message, substr) { + return true + } + } + return false +} From 0c3b666f408824075a7d3f680211bc6672ff8158 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:08:41 +0200 Subject: [PATCH 15/42] l4: decide which conntrack entries a sync leaves stale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A conntrack entry outlives the rule that created it. When a backend goes away, its flows keep being translated to it until they time out: hours for TCP, never for a UDP flow that keeps talking. StaleFlows selects the entries to delete after a commit: translated by this node, towards a VIP the L4 mode knows, to a target it no longer programs. A flow that went through untranslated — a pod on a node that does not announce the VIP — is left alone, and nothing addressed to another IP is ever selected, which keeps the VM mode out of reach. PurgeNeeded tells whether a sync withdrew any translation, so the conntrack table is only dumped when there can be something to remove, and always once after a restart. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/l4/conntrack.go | 103 +++++++++++++++++++++++++++++++++ pkg/l4/conntrack_test.go | 120 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 pkg/l4/conntrack.go create mode 100644 pkg/l4/conntrack_test.go diff --git a/pkg/l4/conntrack.go b/pkg/l4/conntrack.go new file mode 100644 index 0000000..a5b031f --- /dev/null +++ b/pkg/l4/conntrack.go @@ -0,0 +1,103 @@ +package l4 + +import ( + "net/netip" + + v1 "k8s.io/api/core/v1" +) + +// Flow is the part of a conntrack entry the purge decision needs. +type Flow struct { + // Protocol is the IANA protocol number. + Protocol uint8 + // OrigDst is the destination the client used. + OrigDst netip.AddrPort + // ReplySrc is the source of the replies: the backend when this node + // translated the flow, OrigDst otherwise. + ReplySrc netip.AddrPort +} + +// target is one translation the datapath performs. +type target struct { + vip netip.Addr + protocol uint8 + port uint16 + backend netip.AddrPort +} + +func targets(st State) map[target]struct{} { + out := map[target]struct{}{} + for _, r := range st.Rules { + for _, b := range r.Backends { + out[target{r.VIP, protocolNumber(r.Protocol), r.Port, netip.AddrPortFrom(b.IP, b.Port)}] = struct{}{} + } + } + return out +} + +// StaleFlows returns the predicate selecting the conntrack entries to delete +// once cur has been committed, coming from prev (nil after a restart). +// +// A conntrack entry outlives the rule that created it: without the purge, the +// flows of a backend that went away keep being translated to it until they +// time out — hours for TCP, never for an active UDP flow. +// +// An entry is stale when this node translated it (its replies do not come from +// the address the client used), towards a VIP the L4 mode knows, and to a +// target the node no longer programs. Untranslated flows to a VIP — a pod on a +// node that does not announce it, passing through — are left alone, and so is +// everything addressed elsewhere, which keeps the VM mode out of reach. +func StaleFlows(prev *State, cur State) func(Flow) bool { + vips := map[netip.Addr]struct{}{} + for _, st := range []*State{prev, &cur} { + if st == nil { + continue + } + for _, v := range st.VIPs { + vips[v] = struct{}{} + } + for _, r := range st.Rules { + vips[r.VIP] = struct{}{} + } + } + keep := targets(cur) + + return func(f Flow) bool { + if _, ours := vips[f.OrigDst.Addr()]; !ours { + return false + } + if f.ReplySrc == f.OrigDst { + return false + } + _, programmed := keep[target{f.OrigDst.Addr(), f.Protocol, f.OrigDst.Port(), f.ReplySrc}] + return !programmed + } +} + +// PurgeNeeded reports whether moving from prev to cur withdrew a translation, +// which is when conntrack may hold entries StaleFlows would select. After a +// restart (prev nil) the answer is always yes: changes may have happened while +// no instance was watching. +func PurgeNeeded(prev *State, cur State) bool { + if prev == nil { + return true + } + now := targets(cur) + for t := range targets(*prev) { + if _, ok := now[t]; !ok { + return true + } + } + return false +} + +func protocolNumber(p v1.Protocol) uint8 { + switch p { + case v1.ProtocolUDP: + return 17 + case v1.ProtocolSCTP: + return 132 + default: + return 6 + } +} diff --git a/pkg/l4/conntrack_test.go b/pkg/l4/conntrack_test.go new file mode 100644 index 0000000..bac8184 --- /dev/null +++ b/pkg/l4/conntrack_test.go @@ -0,0 +1,120 @@ +package l4 + +import ( + "net/netip" + "testing" + + v1 "k8s.io/api/core/v1" +) + +const ( + tcp = 6 + udp = 17 +) + +func ap(s string) netip.AddrPort { return netip.MustParseAddrPort(s) } + +// translated is a flow this node DNATed from vip to backend. +func translated(proto uint8, vip, backend string) Flow { + return Flow{Protocol: proto, OrigDst: ap(vip), ReplySrc: ap(backend)} +} + +func stateWith(rules ...Rule) State { + st := State{} + for _, r := range rules { + st.VIPs = append(st.VIPs, r.VIP) + st.Ports = append(st.Ports, r.PortKey) + } + st.VIPs = sortedUnique(st.VIPs, netip.Addr.Compare) + st.Rules = rules + return st +} + +func rule(vip string, port uint16, backends ...Backend) Rule { + return Rule{PortKey: key(vip, port), Service: "ns/svc", Backends: backends} +} + +func TestStaleFlows(t *testing.T) { + prev := stateWith( + rule("192.0.2.10", 80, be("10.0.0.1", 8080), be("10.0.0.2", 8080)), + rule("192.0.2.10", 443, be("10.0.0.1", 8443)), + rule("192.0.2.20", 5432, be("10.0.0.3", 5432)), + ) + // 10.0.0.2 went away, port 443 was removed from the service, and the + // 192.0.2.20 service is gone altogether. + cur := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080))) + + stale := StaleFlows(&prev, cur) + + cases := []struct { + name string + flow Flow + want bool + }{ + {"backend still programmed", translated(tcp, "192.0.2.10:80", "10.0.0.1:8080"), false}, + {"backend gone", translated(tcp, "192.0.2.10:80", "10.0.0.2:8080"), true}, + {"port removed", translated(tcp, "192.0.2.10:443", "10.0.0.1:8443"), true}, + {"service removed", translated(tcp, "192.0.2.20:5432", "10.0.0.3:5432"), true}, + {"same backend IP, other port", translated(tcp, "192.0.2.10:80", "10.0.0.1:9090"), true}, + {"protocol not programmed", translated(udp, "192.0.2.10:80", "10.0.0.1:8080"), true}, + // A pod on a node that does not announce the VIP goes through + // untranslated; its flow is not ours to cut. + {"untranslated flow to a VIP", translated(tcp, "192.0.2.10:80", "192.0.2.10:80"), false}, + {"flow to another address", translated(tcp, "198.51.100.1:80", "10.0.0.2:8080"), false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := stale(c.flow); got != c.want { + t.Errorf("stale(%+v) = %v, want %v", c.flow, got, c.want) + } + }) + } +} + +// After a restart there is no previous state: whatever this node translated +// towards a VIP it knows and no longer programs must go — for instance after +// the announcement moved to another node while the pod was down. +func TestStaleFlowsWithoutPreviousState(t *testing.T) { + cur := State{ + VIPs: []netip.Addr{addr("192.0.2.10")}, + Ports: []PortKey{key("192.0.2.10", 80)}, + } + stale := StaleFlows(nil, cur) + if !stale(translated(tcp, "192.0.2.10:80", "10.0.0.1:8080")) { + t.Error("a translation this node no longer programs must be purged") + } + if stale(translated(tcp, "192.0.2.10:80", "192.0.2.10:80")) { + t.Error("an untranslated flow must be kept") + } +} + +func TestPurgeNeeded(t *testing.T) { + base := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080))) + cases := []struct { + name string + prev *State + cur State + want bool + }{ + {"first sync", nil, base, true}, + {"unchanged", &base, base, false}, + {"backend added", &base, stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080), be("10.0.0.2", 8080))), false}, + {"service added", &base, stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080)), rule("192.0.2.11", 80)), false}, + {"backend replaced", &base, stateWith(rule("192.0.2.10", 80, be("10.0.0.2", 8080))), true}, + {"no longer announced", &base, State{VIPs: base.VIPs, Ports: base.Ports}, true}, + {"node IPs only", &base, State{VIPs: base.VIPs, Ports: base.Ports, Rules: base.Rules, NodeIPs: []netip.Addr{addr("10.200.24.11")}}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := PurgeNeeded(c.prev, c.cur); got != c.want { + t.Errorf("PurgeNeeded = %v, want %v", got, c.want) + } + }) + } +} + +func TestProtocolNumber(t *testing.T) { + if protocolNumber(v1.ProtocolTCP) != tcp || protocolNumber(v1.ProtocolUDP) != udp { + t.Error("IANA numbers expected for TCP and UDP") + } +} From e8a6e7be87401d9e030670d7d59023005c84a483 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:16:52 +0200 Subject: [PATCH 16/42] proxy: program the L4 mode's table with nftables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The L4 mode gets its own table, ip cozy_proxy_l4, so it can never touch the VM mode's state, and deleting the table removes the mode entirely. Every sync rebuilds the table as a whole in one transaction. The table is small, nftables applies a batch atomically, and conntrack entries are not tied to rules, so a rebuild cuts nothing. There is no per-element diff to keep right — the VM mode needed a series of fixes for exactly that — and a retry is just the next sync. The table carries a guard on every node, which drops new traffic to a VIP on an undeclared port instead of letting the node route it back to its gateway; a round-robin DNAT with port translation on the announcer; and a masquerade of node sources on translated flows. Flows are recognized by their translated destination rather than by "ct original ip daddr": github.com/google/nftables can only emit the latter with the generic conntrack key, which makes "nft list" abort on the whole ruleset. The tests program a real kernel in a scratch network namespace and compare "nft list" with golden files. They need root and the nft binary, and skip otherwise. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- go.mod | 1 + go.sum | 4 +- pkg/proxy/interface.go | 16 +- pkg/proxy/l4_nft.go | 364 +++++++++++++++++++++++++++ pkg/proxy/l4_nft_test.go | 233 +++++++++++++++++ pkg/proxy/testdata/l4/announcer.nft | 60 +++++ pkg/proxy/testdata/l4/empty.nft | 32 +++ pkg/proxy/testdata/l4/guard-only.nft | 39 +++ 8 files changed, 746 insertions(+), 3 deletions(-) create mode 100644 pkg/proxy/l4_nft.go create mode 100644 pkg/proxy/l4_nft_test.go create mode 100644 pkg/proxy/testdata/l4/announcer.nft create mode 100644 pkg/proxy/testdata/l4/empty.nft create mode 100644 pkg/proxy/testdata/l4/guard-only.nft diff --git a/go.mod b/go.mod index 2be1fb2..120ccaf 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.4 require ( github.com/google/nftables v0.3.0 + github.com/vishvananda/netns v0.0.5 golang.org/x/sys v0.46.0 k8s.io/api v0.36.2 k8s.io/apimachinery v0.36.2 diff --git a/go.sum b/go.sum index 9564083..25087a5 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= -github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= +github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/pkg/proxy/interface.go b/pkg/proxy/interface.go index f7c0e2b..f49fb7a 100644 --- a/pkg/proxy/interface.go +++ b/pkg/proxy/interface.go @@ -1,6 +1,10 @@ package proxy -import corev1 "k8s.io/api/core/v1" +import ( + corev1 "k8s.io/api/core/v1" + + "github.com/cozystack/cozy-proxy/pkg/l4" +) type ProxyProcessor interface { InitRules() error @@ -76,3 +80,13 @@ type PortFilterEntry struct { PodIP string Ports []corev1.ServicePort } + +// L4Datapath programs the L4 LoadBalancer mode. It is independent of +// ProxyProcessor, which drives the VM mode, and never touches its state. +type L4Datapath interface { + // Sync replaces everything the L4 mode programs with st, atomically. + Sync(st l4.State) error + + // Teardown removes everything the L4 mode programs. No-op if absent. + Teardown() error +} diff --git a/pkg/proxy/l4_nft.go b/pkg/proxy/l4_nft.go new file mode 100644 index 0000000..6ac5239 --- /dev/null +++ b/pkg/proxy/l4_nft.go @@ -0,0 +1,364 @@ +package proxy + +import ( + "fmt" + "net" + "net/netip" + "sync" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/google/nftables/userdata" + "golang.org/x/sys/unix" + v1 "k8s.io/api/core/v1" + + "github.com/cozystack/cozy-proxy/pkg/l4" +) + +// L4TableName is the table of the L4 LoadBalancer mode. It is separate from +// the VM mode's "cozy_proxy" table, which it never touches, so deleting it +// removes the L4 mode entirely. +const L4TableName = "cozy_proxy_l4" + +// Chain priorities of the L4 mode. See docs/rfc/l4-loadbalancer-mode.md for how +// they interleave with the VM mode's chains. +var ( + // After conntrack (-200) and the VM mode's ingress_dnat (-150), before + // the DNAT below. + l4GuardPriority = nftables.ChainPriorityRef(*nftables.ChainPriorityMangle + 10) + // Just before the iptables / kube-ovn nat hooks (-100), which then see the + // translated packet. + l4DNATPriority = nftables.ChainPriorityRef(*nftables.ChainPriorityNATDest - 5) + l4MasqPriority = nftables.ChainPriorityRef(*nftables.ChainPriorityNATSource - 5) +) + +// ipsDstNAT is IPS_DST_NAT from linux/netfilter/nf_conntrack_common.h: the +// conntrack status bit set once a flow has been destination-NATed. +const ipsDstNAT = 1 << 5 + +// NFTL4Datapath programs the L4 mode's table with nftables. +// +// Every Sync rebuilds the table as a whole in a single transaction: the table +// is small, nftables applies a batch atomically — a packet sees the old or the +// new ruleset, never a mix — and conntrack entries are not tied to rules. There +// is no incremental diff to get wrong and no stale element to collect, and a +// retry is just the next Sync. +type NFTL4Datapath struct { + mu sync.Mutex + + // ConnOptions are passed to every netlink connection. Tests use them to + // work in a scratch network namespace. + ConnOptions []nftables.ConnOption +} + +// Sync replaces the L4 table with the given state. +func (d *NFTL4Datapath) Sync(st l4.State) error { + d.mu.Lock() + defer d.mu.Unlock() + + conn, err := nftables.New(d.ConnOptions...) + if err != nil { + return fmt.Errorf("could not create nftables connection: %w", err) + } + + t := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: L4TableName} + // Adding the table first makes the deletion valid when it does not exist + // yet, all within the one transaction. + conn.AddTable(t) + conn.DelTable(t) + t = conn.AddTable(t) + + if err := buildL4Table(conn, t, st); err != nil { + return err + } + if err := conn.Flush(); err != nil { + return fmt.Errorf("failed to commit table %s: %w", L4TableName, err) + } + return nil +} + +// Teardown removes the L4 table, if any. It is how the mode is switched off. +func (d *NFTL4Datapath) Teardown() error { + d.mu.Lock() + defer d.mu.Unlock() + + conn, err := nftables.New(d.ConnOptions...) + if err != nil { + return fmt.Errorf("could not create nftables connection: %w", err) + } + + t := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: L4TableName} + conn.AddTable(t) + conn.DelTable(t) + if err := conn.Flush(); err != nil { + return fmt.Errorf("failed to delete table %s: %w", L4TableName, err) + } + return nil +} + +// buildL4Table queues the sets, maps, chains and rules for st into t. +// +// set vips { type ipv4_addr } all VIPs, every node +// set vip_ports { type ipv4_addr . inet_proto . inet_service } declared ports, every node +// set node_ips { type ipv4_addr } masqueraded sources +// set backends { type ipv4_addr . inet_proto . inet_service } local targets, announcer +// map backends--- { type integer : ipv4_addr . inet_service } +// +// chain guard filter prerouting mangle+10 +// ct state established,related accept +// ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop +// chain dnat nat prerouting dstnat-5 (rules on the announcer only) +// ip daddr V tcp dport P dnat ip addr . port to numgen inc mod N map @backends-... +// ip daddr V tcp dport P drop (announced port with no local backend) +// chain masq nat postrouting srcnat-5 +// ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade +func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { + vips := &nftables.Set{Table: t, Name: "vips", KeyType: nftables.TypeIPAddr} + if err := conn.AddSet(vips, addrElements(st.VIPs)); err != nil { + return fmt.Errorf("could not add set %s: %w", vips.Name, err) + } + + portKeyType, err := nftables.ConcatSetType(nftables.TypeIPAddr, nftables.TypeInetProto, nftables.TypeInetService) + if err != nil { + return fmt.Errorf("could not build vip_ports key type: %w", err) + } + vipPorts := &nftables.Set{Table: t, Name: "vip_ports", KeyType: portKeyType, Concatenation: true} + var portElems []nftables.SetElement + for _, pk := range st.Ports { + portElems = append(portElems, nftables.SetElement{ + Key: concatPortKey(net.IP(pk.VIP.AsSlice()), protoByte(pk.Protocol), pk.Port), + }) + } + if err := conn.AddSet(vipPorts, portElems); err != nil { + return fmt.Errorf("could not add set %s: %w", vipPorts.Name, err) + } + + nodeIPs := &nftables.Set{Table: t, Name: "node_ips", KeyType: nftables.TypeIPAddr} + if err := conn.AddSet(nodeIPs, addrElements(st.NodeIPs)); err != nil { + return fmt.Errorf("could not add set %s: %w", nodeIPs.Name, err) + } + + // The translation targets, for the masquerade below to recognize the + // flows this table translated. Matching the flow's original destination + // instead ("ct original ip daddr @vips") is what nft itself would write, + // but github.com/google/nftables v0.3.0 can only emit it with the generic + // conntrack key, which nft then fails to decode: "nft list" aborts on the + // whole ruleset, which is too high a price for every operator on the node. + backends := &nftables.Set{Table: t, Name: "backends", KeyType: portKeyType, Concatenation: true} + var backendElems []nftables.SetElement + seen := map[l4.Backend]bool{} + for _, r := range st.Rules { + for _, b := range r.Backends { + if seen[b] { + continue + } + seen[b] = true + backendElems = append(backendElems, nftables.SetElement{ + Key: concatPortKey(net.IP(b.IP.AsSlice()), protoByte(r.Protocol), b.Port), + }) + } + } + if err := conn.AddSet(backends, backendElems); err != nil { + return fmt.Errorf("could not add set %s: %w", backends.Name, err) + } + + guard := conn.AddChain(&nftables.Chain{ + Name: "guard", + Table: t, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookPrerouting, + Priority: l4GuardPriority, + }) + // ICMP errors about a translated flow (PMTU) are "related": they must reach + // the NAT below, which translates them along with their flow. + conn.AddRule(&nftables.Rule{Table: t, Chain: guard, Exprs: ctStateEstablishedRelatedAccept()}) + // Anything else addressed to a VIP and not declared — another port, ICMP + // echo — is dropped. Left alone it would not be translated, and since the + // VIP is not a local address the node would route it back to its gateway, + // which sends it straight back: a loop until the TTL expires. + conn.AddRule(&nftables.Rule{Table: t, Chain: guard, Exprs: []expr.Any{ + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, + &expr.Lookup{SourceRegister: 1, SetName: vips.Name, SetID: vips.ID}, + &expr.Payload{DestRegister: unix.NFT_REG32_00, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: unix.NFT_REG32_01}, + &expr.Payload{DestRegister: unix.NFT_REG32_02, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, + &expr.Lookup{SourceRegister: unix.NFT_REG32_00, SetName: vipPorts.Name, SetID: vipPorts.ID, Invert: true}, + &expr.Verdict{Kind: expr.VerdictDrop}, + }}) + + dnat := conn.AddChain(&nftables.Chain{ + Name: "dnat", + Table: t, + Type: nftables.ChainTypeNAT, + Hooknum: nftables.ChainHookPrerouting, + Priority: l4DNATPriority, + }) + for _, r := range st.Rules { + exprs := matchFrontend(r.PortKey) + if len(r.Backends) == 0 { + // Announced here with no local ready backend: drop rather than let + // the packet loop through the gateway. + exprs = append(exprs, &expr.Verdict{Kind: expr.VerdictDrop}) + } else { + m, err := addBackendMap(conn, t, r) + if err != nil { + return err + } + exprs = append(exprs, + // Round-robin slot, then slot -> (address . port) into + // NFT_REG32_00 and NFT_REG32_01, which the NAT reads. + &expr.Numgen{Register: 1, Modulus: uint32(len(r.Backends)), Type: unix.NFT_NG_INCREMENTAL}, + &expr.Lookup{SourceRegister: 1, DestRegister: 1, IsDestRegSet: true, SetName: m.Name, SetID: m.ID}, + &expr.NAT{ + Type: expr.NATTypeDestNAT, + Family: unix.NFPROTO_IPV4, + RegAddrMin: 1, + RegProtoMin: unix.NFT_REG32_01, + Specified: true, + }, + ) + } + conn.AddRule(&nftables.Rule{ + Table: t, + Chain: dnat, + Exprs: exprs, + UserData: userdata.AppendString(nil, userdata.TypeComment, r.Service), + }) + } + + masq := conn.AddChain(&nftables.Chain{ + Name: "masq", + Table: t, + Type: nftables.ChainTypeNAT, + Hooknum: nftables.ChainHookPostrouting, + Priority: l4MasqPriority, + }) + // A pod on another node reaches the VIP masqueraded by its own node. Left + // as is, the backend would answer that node IP directly and the overlay + // would carry the reply to the client's node, bypassing this node's + // conntrack: the client then resets the connection. Masquerading the node + // sources pins the reply here. Every other source — the Internet, a VM's + // public IP — is kept. + // + // "ct status dnat" restricts it to translated flows, so a node process + // talking to a backend pod directly (a kubelet probe) keeps its source. + conn.AddRule(&nftables.Rule{Table: t, Chain: masq, Exprs: []expr.Any{ + &expr.Ct{Register: 1, Key: expr.CtKeySTATUS}, + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: binaryutil.NativeEndian.PutUint32(ipsDstNAT), + Xor: binaryutil.NativeEndian.PutUint32(0), + }, + &expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: []byte{0, 0, 0, 0}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, + &expr.Lookup{SourceRegister: 1, SetName: nodeIPs.Name, SetID: nodeIPs.ID}, + &expr.Payload{DestRegister: unix.NFT_REG32_00, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: unix.NFT_REG32_01}, + &expr.Payload{DestRegister: unix.NFT_REG32_02, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, + &expr.Lookup{SourceRegister: unix.NFT_REG32_00, SetName: backends.Name, SetID: backends.ID}, + &expr.Masq{}, + }}) + return nil +} + +// addBackendMap adds the map from round-robin slot to backend for one frontend. +// It is named after the frontend, which is unique, and commented with the +// service it belongs to. +func addBackendMap(conn *nftables.Conn, t *nftables.Table, r l4.Rule) (*nftables.Set, error) { + dataType, err := nftables.ConcatSetType(nftables.TypeIPAddr, nftables.TypeInetService) + if err != nil { + return nil, fmt.Errorf("could not build backend map data type: %w", err) + } + m := &nftables.Set{ + Table: t, + Name: fmt.Sprintf("backends-%s-%s-%d", r.VIP, protoName(r.Protocol), r.Port), + Comment: r.Service, + IsMap: true, + KeyType: nftables.TypeInteger, + DataType: dataType, + // numgen writes the slot in host byte order. nft lists the key type as + // "type 0": it only names an integer key through a "typeof" annotation, + // which github.com/google/nftables cannot write. The elements are listed + // correctly, and the kernel does not look at either. + KeyByteOrder: binaryutil.NativeEndian, + } + elems := make([]nftables.SetElement, 0, len(r.Backends)) + for i, b := range r.Backends { + val := make([]byte, 8) // address, then the port padded to 4 bytes + ip := b.IP.As4() + copy(val[0:4], ip[:]) + val[4] = byte(b.Port >> 8) + val[5] = byte(b.Port) + elems = append(elems, nftables.SetElement{Key: binaryutil.NativeEndian.PutUint32(uint32(i)), Val: val}) + } + if err := conn.AddSet(m, elems); err != nil { + return nil, fmt.Errorf("could not add map %s: %w", m.Name, err) + } + return m, nil +} + +// matchFrontend matches "ip daddr VIP dport PORT". +func matchFrontend(pk l4.PortKey) []expr.Any { + vip := pk.VIP.As4() + return []expr.Any{ + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: vip[:]}, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protoByte(pk.Protocol)}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(pk.Port)}, + } +} + +// ctStateEstablishedRelatedAccept is "ct state established,related accept". +func ctStateEstablishedRelatedAccept() []expr.Any { + return []expr.Any{ + &expr.Ct{Register: 1, SourceRegister: false, Key: expr.CtKeySTATE}, + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED), + Xor: binaryutil.NativeEndian.PutUint32(0), + }, + &expr.Cmp{Op: expr.CmpOpNeq, Register: 1, Data: []byte{0, 0, 0, 0}}, + &expr.Verdict{Kind: expr.VerdictAccept}, + } +} + +func addrElements(addrs []netip.Addr) []nftables.SetElement { + elems := make([]nftables.SetElement, 0, len(addrs)) + for _, a := range addrs { + ip := a.As4() + elems = append(elems, nftables.SetElement{Key: ip[:]}) + } + return elems +} + +func protoByte(p v1.Protocol) byte { + switch p { + case v1.ProtocolUDP: + return unix.IPPROTO_UDP + case v1.ProtocolSCTP: + return unix.IPPROTO_SCTP + default: + return unix.IPPROTO_TCP + } +} + +func protoName(p v1.Protocol) string { + switch p { + case v1.ProtocolUDP: + return "udp" + case v1.ProtocolSCTP: + return "sctp" + default: + return "tcp" + } +} + +// Compile-time assertion that NFTL4Datapath satisfies L4Datapath. +var _ L4Datapath = (*NFTL4Datapath)(nil) diff --git a/pkg/proxy/l4_nft_test.go b/pkg/proxy/l4_nft_test.go new file mode 100644 index 0000000..a14bbc7 --- /dev/null +++ b/pkg/proxy/l4_nft_test.go @@ -0,0 +1,233 @@ +//go:build linux + +package proxy + +import ( + "flag" + "net/netip" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/google/nftables" + "github.com/vishvananda/netns" + v1 "k8s.io/api/core/v1" + + "github.com/cozystack/cozy-proxy/pkg/l4" +) + +// These tests program a real kernel, in a scratch network namespace, and +// compare what `nft list` prints with golden files under testdata/l4. They need +// root and skip otherwise; the golden comparison also needs the nft binary. +// +// Regenerate the golden files with: +// +// go test ./pkg/proxy -run TestL4 -update +var updateGolden = flag.Bool("update", false, "rewrite the golden files under testdata") + +// scratchNetns creates a network namespace that lives as long as the test, so +// nothing here ever touches the host's ruleset. +func scratchNetns(t *testing.T) netns.NsHandle { + t.Helper() + if os.Geteuid() != 0 { + t.Skip("needs root to create a network namespace") + } + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + orig, err := netns.Get() + if err != nil { + t.Fatalf("netns.Get: %v", err) + } + defer orig.Close() + ns, err := netns.New() // also switches this thread into it + if err != nil { + t.Skipf("cannot create a network namespace: %v", err) + } + if err := netns.Set(orig); err != nil { + t.Fatalf("netns.Set: %v", err) + } + t.Cleanup(func() { _ = ns.Close() }) + return ns +} + +func datapathIn(ns netns.NsHandle) *NFTL4Datapath { + return &NFTL4Datapath{ConnOptions: []nftables.ConnOption{nftables.WithNetNSFd(int(ns))}} +} + +// inNetns runs the command inside ns. The child inherits the namespace of the +// thread that forks it, which is locked into ns for the duration. +func inNetns(t *testing.T, ns netns.NsHandle, name string, args ...string) (string, error) { + t.Helper() + runtime.LockOSThread() + defer runtime.UnlockOSThread() + orig, err := netns.Get() + if err != nil { + t.Fatalf("netns.Get: %v", err) + } + defer orig.Close() + if err := netns.Set(ns); err != nil { + t.Fatalf("netns.Set: %v", err) + } + defer func() { + if err := netns.Set(orig); err != nil { + panic(err) // the thread must not go back to the pool in ns + } + }() + out, err := exec.Command(name, args...).CombinedOutput() + return string(out), err +} + +func nftBinary(t *testing.T) string { + t.Helper() + path, err := exec.LookPath("nft") + if err != nil { + t.Skip("nft binary not available") + } + return path +} + +func listL4Table(t *testing.T, ns netns.NsHandle) string { + t.Helper() + out, err := inNetns(t, ns, nftBinary(t), "list", "table", "ip", L4TableName) + if err != nil { + t.Fatalf("nft list table: %v\n%s", err, out) + } + return out +} + +func checkGolden(t *testing.T, name, got string) { + t.Helper() + path := filepath.Join("testdata", "l4", name+".nft") + if *updateGolden { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading golden file (run with -update to create it): %v", err) + } + if got != string(want) { + t.Errorf("%s differs from the golden file\n--- got\n%s\n--- want\n%s", name, got, want) + } +} + +func a(s string) netip.Addr { return netip.MustParseAddr(s) } + +func pk(vip string, proto v1.Protocol, port uint16) l4.PortKey { + return l4.PortKey{VIP: a(vip), Protocol: proto, Port: port} +} + +func bk(ip string, port uint16) l4.Backend { return l4.Backend{IP: a(ip), Port: port} } + +// announcerState is what the announcer of two services programs: one with two +// ports and two local backends, one announced here with no local backend. +var announcerState = l4.State{ + VIPs: []netip.Addr{a("192.0.2.10"), a("192.0.2.20"), a("192.0.2.30")}, + Ports: []l4.PortKey{ + pk("192.0.2.10", v1.ProtocolTCP, 80), + pk("192.0.2.10", v1.ProtocolTCP, 443), + pk("192.0.2.20", v1.ProtocolTCP, 5432), + }, + NodeIPs: []netip.Addr{a("10.200.24.11"), a("10.200.24.12"), a("10.200.24.13")}, + Rules: []l4.Rule{ + {PortKey: pk("192.0.2.10", v1.ProtocolTCP, 80), Service: "tenant-a/ingress", + Backends: []l4.Backend{bk("10.244.0.10", 8080), bk("10.244.0.11", 8080)}}, + {PortKey: pk("192.0.2.10", v1.ProtocolTCP, 443), Service: "tenant-a/ingress", + Backends: []l4.Backend{bk("10.244.0.10", 8443), bk("10.244.0.11", 8443)}}, + {PortKey: pk("192.0.2.20", v1.ProtocolTCP, 5432), Service: "tenant-b/postgres"}, + }, +} + +// guardOnlyState is the same cluster seen from a node that announces nothing: +// it guards every VIP and translates none. +var guardOnlyState = l4.State{ + VIPs: announcerState.VIPs, + Ports: announcerState.Ports, + NodeIPs: announcerState.NodeIPs, +} + +func TestL4SyncGolden(t *testing.T) { + cases := []struct { + name string + state l4.State + }{ + {"announcer", announcerState}, + {"guard-only", guardOnlyState}, + {"empty", l4.State{}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ns := scratchNetns(t) + if err := datapathIn(ns).Sync(c.state); err != nil { + t.Fatalf("Sync: %v", err) + } + checkGolden(t, c.name, listL4Table(t, ns)) + }) + } +} + +// A sync replaces the table as a whole: nothing from the previous state may +// survive, a backend map in particular. +func TestL4SyncReplacesPreviousState(t *testing.T) { + ns := scratchNetns(t) + dp := datapathIn(ns) + if err := dp.Sync(announcerState); err != nil { + t.Fatalf("first Sync: %v", err) + } + if err := dp.Sync(guardOnlyState); err != nil { + t.Fatalf("second Sync: %v", err) + } + got := listL4Table(t, ns) + if strings.Contains(got, "backends-") || strings.Contains(got, "dnat") && strings.Contains(got, "numgen") { + t.Errorf("state of the first sync survived the second:\n%s", got) + } + checkGolden(t, "guard-only", got) +} + +// Teardown removes the table, is idempotent, and never touches another table +// — the VM mode's in particular. +func TestL4TeardownLeavesOtherTablesAlone(t *testing.T) { + ns := scratchNetns(t) + dp := datapathIn(ns) + + conn, err := nftables.New(nftables.WithNetNSFd(int(ns))) + if err != nil { + t.Fatalf("nftables.New: %v", err) + } + conn.AddTable(&nftables.Table{Family: nftables.TableFamilyIPv4, Name: "cozy_proxy"}) + if err := conn.Flush(); err != nil { + t.Fatalf("creating the VM mode's table: %v", err) + } + + if err := dp.Teardown(); err != nil { + t.Fatalf("Teardown without a table: %v", err) + } + if err := dp.Sync(announcerState); err != nil { + t.Fatalf("Sync: %v", err) + } + if err := dp.Teardown(); err != nil { + t.Fatalf("Teardown: %v", err) + } + + tables, err := conn.ListTablesOfFamily(nftables.TableFamilyIPv4) + if err != nil { + t.Fatalf("ListTables: %v", err) + } + var names []string + for _, tb := range tables { + names = append(names, tb.Name) + } + if len(names) != 1 || names[0] != "cozy_proxy" { + t.Errorf("tables after Teardown = %v, want only cozy_proxy", names) + } +} diff --git a/pkg/proxy/testdata/l4/announcer.nft b/pkg/proxy/testdata/l4/announcer.nft new file mode 100644 index 0000000..ef5fb32 --- /dev/null +++ b/pkg/proxy/testdata/l4/announcer.nft @@ -0,0 +1,60 @@ +table ip cozy_proxy_l4 { + set vips { + type ipv4_addr + elements = { 192.0.2.10, 192.0.2.20, + 192.0.2.30 } + } + + set vip_ports { + type ipv4_addr . inet_proto . inet_service + elements = { 192.0.2.20 . tcp . 5432, + 192.0.2.10 . tcp . 80, + 192.0.2.10 . tcp . 443 } + } + + set node_ips { + type ipv4_addr + elements = { 10.200.24.11, 10.200.24.12, + 10.200.24.13 } + } + + set backends { + type ipv4_addr . inet_proto . inet_service + elements = { 10.244.0.10 . tcp . 8080, + 10.244.0.11 . tcp . 8080, + 10.244.0.10 . tcp . 8443, + 10.244.0.11 . tcp . 8443 } + } + + map backends-192.0.2.10-tcp-80 { + type 0 : ipv4_addr . inet_service + comment "tenant-a/ingress" + elements = { 0 : 10.244.0.10 . 8080, + 1 : 10.244.0.11 . 8080 } + } + + map backends-192.0.2.10-tcp-443 { + type 0 : ipv4_addr . inet_service + comment "tenant-a/ingress" + elements = { 0 : 10.244.0.10 . 8443, + 1 : 10.244.0.11 . 8443 } + } + + chain guard { + type filter hook prerouting priority mangle + 10; policy accept; + ct state established,related accept + ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop + } + + chain dnat { + type nat hook prerouting priority dstnat - 5; policy accept; + ip daddr 192.0.2.10 tcp dport 80 dnat ip to numgen inc mod 2 map @backends-192.0.2.10-tcp-80 comment "tenant-a/ingress" + ip daddr 192.0.2.10 tcp dport 443 dnat ip to numgen inc mod 2 map @backends-192.0.2.10-tcp-443 comment "tenant-a/ingress" + ip daddr 192.0.2.20 tcp dport 5432 drop comment "tenant-b/postgres" + } + + chain masq { + type nat hook postrouting priority srcnat - 5; policy accept; + ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade + } +} diff --git a/pkg/proxy/testdata/l4/empty.nft b/pkg/proxy/testdata/l4/empty.nft new file mode 100644 index 0000000..9408bd7 --- /dev/null +++ b/pkg/proxy/testdata/l4/empty.nft @@ -0,0 +1,32 @@ +table ip cozy_proxy_l4 { + set vips { + type ipv4_addr + } + + set vip_ports { + type ipv4_addr . inet_proto . inet_service + } + + set node_ips { + type ipv4_addr + } + + set backends { + type ipv4_addr . inet_proto . inet_service + } + + chain guard { + type filter hook prerouting priority mangle + 10; policy accept; + ct state established,related accept + ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop + } + + chain dnat { + type nat hook prerouting priority dstnat - 5; policy accept; + } + + chain masq { + type nat hook postrouting priority srcnat - 5; policy accept; + ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade + } +} diff --git a/pkg/proxy/testdata/l4/guard-only.nft b/pkg/proxy/testdata/l4/guard-only.nft new file mode 100644 index 0000000..0ac1af7 --- /dev/null +++ b/pkg/proxy/testdata/l4/guard-only.nft @@ -0,0 +1,39 @@ +table ip cozy_proxy_l4 { + set vips { + type ipv4_addr + elements = { 192.0.2.10, 192.0.2.20, + 192.0.2.30 } + } + + set vip_ports { + type ipv4_addr . inet_proto . inet_service + elements = { 192.0.2.20 . tcp . 5432, + 192.0.2.10 . tcp . 80, + 192.0.2.10 . tcp . 443 } + } + + set node_ips { + type ipv4_addr + elements = { 10.200.24.11, 10.200.24.12, + 10.200.24.13 } + } + + set backends { + type ipv4_addr . inet_proto . inet_service + } + + chain guard { + type filter hook prerouting priority mangle + 10; policy accept; + ct state established,related accept + ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop + } + + chain dnat { + type nat hook prerouting priority dstnat - 5; policy accept; + } + + chain masq { + type nat hook postrouting priority srcnat - 5; policy accept; + ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade + } +} From cb5bc01f42ad53481c9ff682dd9959bd3d62edb5 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:18:40 +0200 Subject: [PATCH 17/42] proxy: purge conntrack entries over netlink The L4 mode decides which conntrack entries a sync left stale; this is the part that deletes them. The kernel cannot filter on "translated to this backend", so the table is dumped and each entry matched against the predicate, through vishvananda/netlink, which kube-proxy, Cilium and kube-ovn use for the same job. The test creates translated entries in a scratch network namespace and checks that only the selected one goes. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- go.mod | 1 + go.sum | 4 ++ pkg/proxy/conntrack.go | 69 ++++++++++++++++++++ pkg/proxy/conntrack_test.go | 124 ++++++++++++++++++++++++++++++++++++ pkg/proxy/interface.go | 7 ++ 5 files changed, 205 insertions(+) create mode 100644 pkg/proxy/conntrack.go create mode 100644 pkg/proxy/conntrack_test.go diff --git a/go.mod b/go.mod index 120ccaf..f0ad06c 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.4 require ( github.com/google/nftables v0.3.0 + github.com/vishvananda/netlink v1.3.1 github.com/vishvananda/netns v0.0.5 golang.org/x/sys v0.46.0 k8s.io/api v0.36.2 diff --git a/go.sum b/go.sum index 25087a5..7057511 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= +github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -142,6 +144,8 @@ golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= diff --git a/pkg/proxy/conntrack.go b/pkg/proxy/conntrack.go new file mode 100644 index 0000000..5a51c3f --- /dev/null +++ b/pkg/proxy/conntrack.go @@ -0,0 +1,69 @@ +package proxy + +import ( + "fmt" + "net/netip" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "github.com/cozystack/cozy-proxy/pkg/l4" +) + +// NetlinkConntrack deletes conntrack entries over netlink. +type NetlinkConntrack struct { + // Handle, when set, is used instead of the default one. Tests use it to + // work in a scratch network namespace. + Handle *netlink.Handle +} + +// Purge deletes the IPv4 conntrack entries stale selects. The kernel has no +// filter for "translated to this backend", so the table is dumped and matched +// here, which is why the caller only purges when a translation was withdrawn. +func (c *NetlinkConntrack) Purge(stale func(l4.Flow) bool) (uint, error) { + h := c.Handle + if h == nil { + var err error + if h, err = netlink.NewHandle(unix.NETLINK_NETFILTER); err != nil { + return 0, fmt.Errorf("could not open a netfilter netlink handle: %w", err) + } + defer h.Close() + } + n, err := h.ConntrackDeleteFilters(netlink.ConntrackTable, unix.AF_INET, flowFilter(stale)) + if err != nil { + return n, fmt.Errorf("conntrack purge: %w", err) + } + return n, nil +} + +// flowFilter adapts a predicate on l4.Flow to the netlink filter interface. +type flowFilter func(l4.Flow) bool + +func (f flowFilter) MatchConntrackFlow(flow *netlink.ConntrackFlow) bool { + lf, ok := toFlow(flow) + return ok && f(lf) +} + +// toFlow extracts what the purge decision needs: the destination the client +// used, and where the replies come from — the backend, when translated. +func toFlow(f *netlink.ConntrackFlow) (l4.Flow, bool) { + if f == nil { + return l4.Flow{}, false + } + dst, ok := netip.AddrFromSlice(f.Forward.DstIP) + if !ok { + return l4.Flow{}, false + } + src, ok := netip.AddrFromSlice(f.Reverse.SrcIP) + if !ok { + return l4.Flow{}, false + } + return l4.Flow{ + Protocol: f.Forward.Protocol, + OrigDst: netip.AddrPortFrom(dst.Unmap(), f.Forward.DstPort), + ReplySrc: netip.AddrPortFrom(src.Unmap(), f.Reverse.SrcPort), + }, true +} + +// Compile-time assertion that NetlinkConntrack satisfies ConntrackPurger. +var _ ConntrackPurger = (*NetlinkConntrack)(nil) diff --git a/pkg/proxy/conntrack_test.go b/pkg/proxy/conntrack_test.go new file mode 100644 index 0000000..c9f929a --- /dev/null +++ b/pkg/proxy/conntrack_test.go @@ -0,0 +1,124 @@ +//go:build linux + +package proxy + +import ( + "net" + "net/netip" + "testing" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" + + "github.com/cozystack/cozy-proxy/pkg/l4" +) + +// conntrackIn returns a netlink handle working in ns. +func conntrackIn(t *testing.T, ns netns.NsHandle) *netlink.Handle { + t.Helper() + h, err := netlink.NewHandleAt(ns) + if err != nil { + t.Fatalf("netlink.NewHandleAt: %v", err) + } + t.Cleanup(h.Close) + return h +} + +// createFlow adds a TCP entry translated from client -> origDst to +// replySrc -> client, the way a DNAT leaves it. +func createFlow(t *testing.T, h *netlink.Handle, client, origDst, replySrc string) { + t.Helper() + c := netip.MustParseAddrPort(client) + d := netip.MustParseAddrPort(origDst) + r := netip.MustParseAddrPort(replySrc) + flow := &netlink.ConntrackFlow{ + FamilyType: unix.AF_INET, + Forward: netlink.IPTuple{ + Protocol: unix.IPPROTO_TCP, + SrcIP: net.IP(c.Addr().AsSlice()), SrcPort: c.Port(), + DstIP: net.IP(d.Addr().AsSlice()), DstPort: d.Port(), + }, + Reverse: netlink.IPTuple{ + Protocol: unix.IPPROTO_TCP, + SrcIP: net.IP(r.Addr().AsSlice()), SrcPort: r.Port(), + DstIP: net.IP(c.Addr().AsSlice()), DstPort: c.Port(), + }, + TimeOut: 300, + ProtoInfo: &netlink.ProtoInfoTCP{State: 3}, // TCP_CONNTRACK_ESTABLISHED + } + if err := h.ConntrackCreate(netlink.ConntrackTable, unix.AF_INET, flow); err != nil { + t.Fatalf("ConntrackCreate %s -> %s: %v", client, origDst, err) + } +} + +func listFlows(t *testing.T, h *netlink.Handle) []l4.Flow { + t.Helper() + flows, err := h.ConntrackTableList(netlink.ConntrackTable, unix.AF_INET) + if err != nil { + t.Fatalf("ConntrackTableList: %v", err) + } + var out []l4.Flow + for _, f := range flows { + if lf, ok := toFlow(f); ok { + out = append(out, lf) + } + } + return out +} + +func TestNetlinkConntrackPurgeDeletesSelectedFlowsOnly(t *testing.T) { + ns := scratchNetns(t) + h := conntrackIn(t, ns) + + createFlow(t, h, "198.51.100.7:40001", "192.0.2.10:80", "10.244.0.10:8080") // backend kept + createFlow(t, h, "198.51.100.7:40002", "192.0.2.10:80", "10.244.0.11:8080") // backend gone + createFlow(t, h, "198.51.100.7:40003", "203.0.113.5:443", "10.244.0.11:8443") // not a VIP + + gone := netip.MustParseAddrPort("10.244.0.11:8080") + purger := &NetlinkConntrack{Handle: h} + n, err := purger.Purge(func(f l4.Flow) bool { return f.ReplySrc == gone }) + if err != nil { + t.Fatalf("Purge: %v", err) + } + if n != 1 { + t.Errorf("Purge deleted %d flows, want 1", n) + } + + left := listFlows(t, h) + if len(left) != 2 { + t.Fatalf("flows left = %+v, want 2", left) + } + for _, f := range left { + if f.ReplySrc == gone { + t.Errorf("the selected flow survived: %+v", f) + } + } +} + +// The flow handed to the predicate must carry the translation: the original +// destination and the source of the replies. +func TestToFlow(t *testing.T) { + f := &netlink.ConntrackFlow{ + FamilyType: unix.AF_INET, + Forward: netlink.IPTuple{Protocol: unix.IPPROTO_TCP, + SrcIP: net.ParseIP("198.51.100.7"), SrcPort: 40001, + DstIP: net.ParseIP("192.0.2.10"), DstPort: 80}, + Reverse: netlink.IPTuple{Protocol: unix.IPPROTO_TCP, + SrcIP: net.ParseIP("10.244.0.10"), SrcPort: 8080, + DstIP: net.ParseIP("100.64.0.4"), DstPort: 21653}, + } + got, ok := toFlow(f) + want := l4.Flow{ + Protocol: unix.IPPROTO_TCP, + OrigDst: netip.MustParseAddrPort("192.0.2.10:80"), + ReplySrc: netip.MustParseAddrPort("10.244.0.10:8080"), + } + if !ok || got != want { + t.Errorf("toFlow = %+v, %v, want %+v", got, ok, want) + } + + if _, ok := toFlow(&netlink.ConntrackFlow{}); ok { + t.Error("a flow without addresses must be skipped") + } +} diff --git a/pkg/proxy/interface.go b/pkg/proxy/interface.go index f49fb7a..eddd7b4 100644 --- a/pkg/proxy/interface.go +++ b/pkg/proxy/interface.go @@ -90,3 +90,10 @@ type L4Datapath interface { // Teardown removes everything the L4 mode programs. No-op if absent. Teardown() error } + +// ConntrackPurger deletes conntrack entries. +type ConntrackPurger interface { + // Purge deletes the IPv4 entries stale selects and returns how many + // were deleted. + Purge(stale func(l4.Flow) bool) (uint, error) +} From 98c6c95d78f581976817775bfc7554045cd74c25 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:20:40 +0200 Subject: [PATCH 18/42] proxy: send real traffic through the L4 table in tests The golden files say what the kernel holds, not what it does with a packet. These tests build a client, an announcer and a backend node out of three network namespaces, program the announcer, and connect through the VIP, which exists nowhere but in the table, like a MetalLB VIP that Cilium released. They check the port translation and the round-robin between two backends, that a client keeps its own address unless it is a node, that a node source is masqueraded to the announcer, that an undeclared port or a port without backend is dropped silently instead of being routed on, and that withdrawing a backend purges its flows from the announcer's conntrack. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/proxy/l4_datapath_test.go | 320 ++++++++++++++++++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 pkg/proxy/l4_datapath_test.go diff --git a/pkg/proxy/l4_datapath_test.go b/pkg/proxy/l4_datapath_test.go new file mode 100644 index 0000000..70c745c --- /dev/null +++ b/pkg/proxy/l4_datapath_test.go @@ -0,0 +1,320 @@ +//go:build linux + +package proxy + +import ( + "bufio" + "errors" + "net" + "net/netip" + "os" + "runtime" + "strings" + "testing" + "time" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + v1 "k8s.io/api/core/v1" + + "github.com/cozystack/cozy-proxy/pkg/l4" +) + +// These tests send real traffic through the L4 table. Three network +// namespaces stand in for a client, the announcer and the node hosting the +// backends: +// +// client 10.0.1.2 ──── 10.0.1.1 router 10.0.2.1 ──── 10.0.2.2, 10.0.2.3 server +// (cozy_proxy_l4) +// +// The VIP 192.0.2.10 is not configured anywhere: like a MetalLB L2 VIP that +// Cilium released, it only exists through the table. + +const ( + testVIP = "192.0.2.10" + backendPort = 8080 + routerToSrv = "10.0.2.1" + clientAddr = "10.0.1.2" + backendA = "10.0.2.2" + backendB = "10.0.2.3" + dialTimeout = 500 * time.Millisecond + serverTimeout = 2 * time.Second +) + +type topology struct { + client, router, server netns.NsHandle +} + +// doIn runs f on a thread switched into ns. Sockets created by f stay in ns. +func doIn(t *testing.T, ns netns.NsHandle, f func() error) error { + t.Helper() + runtime.LockOSThread() + defer runtime.UnlockOSThread() + orig, err := netns.Get() + if err != nil { + t.Fatalf("netns.Get: %v", err) + } + defer orig.Close() + if err := netns.Set(ns); err != nil { + t.Fatalf("netns.Set: %v", err) + } + defer func() { + if err := netns.Set(orig); err != nil { + panic(err) // the thread must not go back to the pool in ns + } + }() + return f() +} + +func must(t *testing.T, what string, err error) { + t.Helper() + if err != nil { + t.Fatalf("%s: %v", what, err) + } +} + +func newTopology(t *testing.T) topology { + t.Helper() + tp := topology{client: scratchNetns(t), router: scratchNetns(t), server: scratchNetns(t)} + + h := map[netns.NsHandle]*netlink.Handle{} + for _, ns := range []netns.NsHandle{tp.client, tp.router, tp.server} { + nh, err := netlink.NewHandleAt(ns) + must(t, "netlink.NewHandleAt", err) + t.Cleanup(nh.Close) + h[ns] = nh + lo, err := nh.LinkByName("lo") + must(t, "lo", err) + must(t, "lo up", nh.LinkSetUp(lo)) + } + + // One veth per side, created in the router and moved into its peer. + link := func(peerNS netns.NsHandle, routerEnd, peerEnd, routerIP, peerIPs string) { + must(t, "veth "+routerEnd, h[tp.router].LinkAdd(&netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{Name: routerEnd}, PeerName: peerEnd, + })) + peer, err := h[tp.router].LinkByName(peerEnd) + must(t, "peer "+peerEnd, err) + must(t, "move "+peerEnd, h[tp.router].LinkSetNsFd(peer, int(peerNS))) + + r, err := h[tp.router].LinkByName(routerEnd) + must(t, "router end", err) + must(t, "router addr", h[tp.router].AddrAdd(r, mustAddr(t, routerIP+"/24"))) + must(t, "router up", h[tp.router].LinkSetUp(r)) + + p, err := h[peerNS].LinkByName(peerEnd) + must(t, "peer end", err) + for _, ip := range strings.Split(peerIPs, ",") { + must(t, "peer addr", h[peerNS].AddrAdd(p, mustAddr(t, ip+"/24"))) + } + must(t, "peer up", h[peerNS].LinkSetUp(p)) + must(t, "peer default route", h[peerNS].RouteAdd(&netlink.Route{ + LinkIndex: p.Attrs().Index, Gw: net.ParseIP(routerIP), + })) + } + link(tp.client, "rc", "cr", "10.0.1.1", clientAddr) + link(tp.server, "rs", "sr", routerToSrv, backendA+","+backendB) + + must(t, "ip_forward", doIn(t, tp.router, func() error { + return os.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1"), 0o644) + })) + return tp +} + +func mustAddr(t *testing.T, cidr string) *netlink.Addr { + t.Helper() + a, err := netlink.ParseAddr(cidr) + must(t, "ParseAddr "+cidr, err) + return a +} + +// serve answers every connection on the backends with " ", so +// the client learns which backend took it and which source it saw. +func serve(t *testing.T, ns netns.NsHandle) { + t.Helper() + var ln net.Listener + must(t, "listen", doIn(t, ns, func() error { + var err error + ln, err = net.Listen("tcp4", "0.0.0.0:8080") + return err + })) + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + _, _ = c.Write([]byte(c.RemoteAddr().String() + " " + c.LocalAddr().String() + "\n")) + _ = c.Close() + } + }() +} + +type answer struct { + seenSource netip.Addr // the client address as the backend saw it + backend netip.Addr // the backend that took the connection +} + +func dial(t *testing.T, ns netns.NsHandle, addr string) (answer, error) { + t.Helper() + var conn net.Conn + err := doIn(t, ns, func() error { + var err error + conn, err = net.DialTimeout("tcp4", addr, dialTimeout) + return err + }) + if err != nil { + return answer{}, err + } + defer conn.Close() + _ = conn.SetReadDeadline(time.Now().Add(serverTimeout)) + line, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + return answer{}, err + } + f := strings.Fields(line) + if len(f) != 2 { + return answer{}, errors.New("malformed answer " + line) + } + return answer{ + seenSource: netip.MustParseAddrPort(f[0]).Addr(), + backend: netip.MustParseAddrPort(f[1]).Addr(), + }, nil +} + +// routerState is what the announcer programs for one service with two local +// backends on port 80 -> 8080. +func routerState(nodeIPs ...string) l4.State { + st := l4.State{ + VIPs: []netip.Addr{a(testVIP)}, + Ports: []l4.PortKey{pk(testVIP, v1.ProtocolTCP, 80)}, + Rules: []l4.Rule{{ + PortKey: pk(testVIP, v1.ProtocolTCP, 80), + Service: "ns/svc", + Backends: []l4.Backend{bk(backendA, backendPort), bk(backendB, backendPort)}, + }}, + } + for _, ip := range nodeIPs { + st.NodeIPs = append(st.NodeIPs, a(ip)) + } + return st +} + +func TestL4DatapathTranslatesAndBalances(t *testing.T) { + tp := newTopology(t) + serve(t, tp.server) + must(t, "Sync", datapathIn(tp.router).Sync(routerState())) + + seen := map[netip.Addr]bool{} + for i := 0; i < 2; i++ { + ans, err := dial(t, tp.client, testVIP+":80") + if err != nil { + t.Fatalf("connection %d to the VIP: %v", i, err) + } + // A client that is not a node keeps its address: this is what lets an + // Internet client, or a VM with its public IP, be seen as itself. + if ans.seenSource != a(clientAddr) { + t.Errorf("backend saw %s, want the client's own address %s", ans.seenSource, clientAddr) + } + seen[ans.backend] = true + } + if !seen[a(backendA)] || !seen[a(backendB)] { + t.Errorf("two connections must reach both backends, got %v", seen) + } +} + +// A client that is a node — a pod masqueraded by its own node — must be +// masqueraded again, so the reply comes back through the announcer. +func TestL4DatapathMasqueradesNodeSources(t *testing.T) { + tp := newTopology(t) + serve(t, tp.server) + must(t, "Sync", datapathIn(tp.router).Sync(routerState(clientAddr))) + + ans, err := dial(t, tp.client, testVIP+":80") + if err != nil { + t.Fatalf("connection to the VIP: %v", err) + } + if ans.seenSource != a(routerToSrv) { + t.Errorf("backend saw %s, want the announcer's address %s", ans.seenSource, routerToSrv) + } +} + +// Without the guard, the announcer routes an undeclared port onwards — on a +// real node back to the gateway, which loops it. Here the router has no route +// for the VIP, so an unguarded packet would come back as an ICMP error, and +// the client would fail fast instead of timing out. +func TestL4DatapathGuardDropsUndeclaredPorts(t *testing.T) { + tp := newTopology(t) + serve(t, tp.server) + dp := datapathIn(tp.router) + + _, err := dial(t, tp.client, testVIP+":81") + if err == nil || isTimeout(err) { + t.Fatalf("without the table, the router must reject the VIP outright, got %v", err) + } + + must(t, "Sync", dp.Sync(routerState())) + _, err = dial(t, tp.client, testVIP+":81") + if !isTimeout(err) { + t.Errorf("an undeclared port must be dropped silently, got %v", err) + } +} + +// An announced port with no local backend is dropped as well. +func TestL4DatapathDropsPortWithoutBackend(t *testing.T) { + tp := newTopology(t) + serve(t, tp.server) + st := routerState() + st.Rules[0].Backends = nil + must(t, "Sync", datapathIn(tp.router).Sync(st)) + + if _, err := dial(t, tp.client, testVIP+":80"); !isTimeout(err) { + t.Errorf("a port without backend must be dropped, got %v", err) + } +} + +// After a backend is withdrawn, its existing flows are purged from the +// announcer's conntrack, and new connections only reach the remaining one. +func TestL4DatapathPurgesWithdrawnBackend(t *testing.T) { + tp := newTopology(t) + serve(t, tp.server) + dp := datapathIn(tp.router) + ct := &NetlinkConntrack{Handle: conntrackIn(t, tp.router)} + + prev := routerState() + must(t, "Sync", dp.Sync(prev)) + for i := 0; i < 2; i++ { + if _, err := dial(t, tp.client, testVIP+":80"); err != nil { + t.Fatalf("connection %d: %v", i, err) + } + } + + cur := routerState() + cur.Rules[0].Backends = []l4.Backend{bk(backendA, backendPort)} + must(t, "Sync", dp.Sync(cur)) + if _, err := ct.Purge(l4.StaleFlows(&prev, cur)); err != nil { + t.Fatalf("Purge: %v", err) + } + + gone := netip.AddrPortFrom(a(backendB), backendPort) + for _, f := range listFlows(t, ct.Handle) { + if f.ReplySrc == gone { + t.Errorf("a flow towards the withdrawn backend survived: %+v", f) + } + } + for i := 0; i < 2; i++ { + ans, err := dial(t, tp.client, testVIP+":80") + if err != nil { + t.Fatalf("connection after the withdrawal: %v", err) + } + if ans.backend != a(backendA) { + t.Errorf("connection reached %s, want only %s", ans.backend, backendA) + } + } +} + +func isTimeout(err error) bool { + var ne net.Error + return errors.As(err, &ne) && ne.Timeout() +} From a7237b6dbb8a1d7ce0b44a048bdc1d470232d189 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:26:23 +0200 Subject: [PATCH 19/42] controllers: reconcile the L4 mode from the informer caches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The L4 mode gets its own controller, independent of the VM mode's. It watches Services, EndpointSlices, Nodes and MetalLB's ServiceL2Status, the latter filtered on this node's label, since all a node needs to know is which VIPs it announces. Reconciliation is level-triggered. Events only mark the state dirty; a single loop recomputes the whole desired state from the caches, commits it if it changed, and purges the conntrack entries the commit left stale. Bursts are coalesced, a failed pass is retried a few seconds later, and a periodic resync re-applies the table so one removed by hand comes back. A failed purge stays pending, and keeps the VIPs of the services removed in the meantime within reach. The first pass waits for complete caches: programming from a partial view would withdraw live services. Until then the table the previous instance left keeps forwarding, and on shutdown the table stays in place, so a DaemonSet rollout does not cut traffic. Start never fails: the VM mode runs in the same manager, and an L4 problem — no NODE_NAME, a missing MetalLB CRD — must not take it down. It is logged and the mode stays idle or keeps retrying. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/controllers/l4_controller.go | 311 +++++++++++++++++++++ pkg/controllers/l4_controller_test.go | 376 ++++++++++++++++++++++++++ 2 files changed, 687 insertions(+) create mode 100644 pkg/controllers/l4_controller.go create mode 100644 pkg/controllers/l4_controller_test.go diff --git a/pkg/controllers/l4_controller.go b/pkg/controllers/l4_controller.go new file mode 100644 index 0000000..42e8ced --- /dev/null +++ b/pkg/controllers/l4_controller.go @@ -0,0 +1,311 @@ +package controllers + +import ( + "context" + "fmt" + "net/netip" + "reflect" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/cozystack/cozy-proxy/pkg/l4" + nat "github.com/cozystack/cozy-proxy/pkg/proxy" +) + +var l4log = log.WithName("l4") + +// serviceL2StatusGVR is MetalLB's record of which node announces a service in +// L2 mode. Each speaker writes one per service it announces, labelled with its +// node. +var serviceL2StatusGVR = schema.GroupVersionResource{ + Group: "metallb.io", + Version: "v1beta1", + Resource: "servicel2statuses", +} + +const metallbNodeLabel = "metallb.io/node" + +const ( + defaultL4MinSyncInterval = 250 * time.Millisecond + defaultL4ResyncInterval = time.Minute + defaultL4RetryInterval = 5 * time.Second +) + +// L4Controller runs the L4 LoadBalancer mode: it keeps this node's L4 table +// in line with the cluster. It shares nothing with ServicesController, which +// runs the VM mode, but the process. +// +// Reconciliation is level-triggered. Events only mark the state dirty; a +// single loop recomputes the whole desired state from the informer caches and +// hands it to the datapath, which replaces its table atomically. A failure is +// simply retried by a later pass. +type L4Controller struct { + Clientset kubernetes.Interface + Dynamic dynamic.Interface + // NodeName is the node this instance programs. Required: without it the + // mode cannot tell which backends are local, and programs nothing. + NodeName string + Datapath nat.L4Datapath + Conntrack nat.ConntrackPurger + + // MinSyncInterval coalesces bursts of events into one pass. + MinSyncInterval time.Duration + // ResyncInterval re-applies the state even without events, so that a + // table removed or altered by hand comes back. + ResyncInterval time.Duration + // RetryInterval is how soon a failed pass is retried. + RetryInterval time.Duration + + dirty chan struct{} + input func() l4.Input + + // applied is the last state the datapath committed; nil until the first + // commit of this process. + applied *l4.State + // purgePending is set when a commit withdrew translations whose + // conntrack entries have not been purged yet, and purgeVIPs collects the + // VIPs those entries may target, removed services included. + purgePending bool + purgeVIPs map[netip.Addr]struct{} + // notices holds what was last logged per service, so a notice is logged + // when it changes rather than on every pass. + notices map[string]string +} + +// Start runs the controller until ctx ends. +// +// It never returns an error: the VM mode runs in the same manager, and a +// failure of the L4 mode must not take it down. Problems are logged and +// retried instead. +func (c *L4Controller) Start(ctx context.Context) error { + if c.NodeName == "" { + l4log.Error(nil, "NODE_NAME is not set; the L4 mode cannot tell which backends are local and stays idle") + <-ctx.Done() + return nil + } + l4log.Info("starting the L4 LoadBalancer mode", "node", c.NodeName) + + factory := informers.NewSharedInformerFactory(c.Clientset, 0) + svcInformer := factory.Core().V1().Services() + sliceInformer := factory.Discovery().V1().EndpointSlices() + nodeInformer := factory.Core().V1().Nodes() + + // Only this node's statuses: that is all the mode needs to know. + dynFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(c.Dynamic, 0, metav1.NamespaceAll, + func(o *metav1.ListOptions) { o.LabelSelector = metallbNodeLabel + "=" + c.NodeName }) + l2Informer := dynFactory.ForResource(serviceL2StatusGVR) + + c.dirty = make(chan struct{}, 1) + handler := cache.ResourceEventHandlerFuncs{ + AddFunc: func(interface{}) { c.markDirty() }, + UpdateFunc: func(interface{}, interface{}) { c.markDirty() }, + DeleteFunc: func(interface{}) { c.markDirty() }, + } + for _, inf := range []cache.SharedIndexInformer{ + svcInformer.Informer(), sliceInformer.Informer(), nodeInformer.Informer(), l2Informer.Informer(), + } { + if _, err := inf.AddEventHandler(handler); err != nil { + l4log.Error(err, "could not register an event handler; the L4 mode stays idle") + <-ctx.Done() + return nil + } + } + + factory.Start(ctx.Done()) + dynFactory.Start(ctx.Done()) + + // The table the previous instance left keeps forwarding until the first + // pass, which needs complete caches: programming from a partial view would + // withdraw live services. + synced := []cache.InformerSynced{ + svcInformer.Informer().HasSynced, + sliceInformer.Informer().HasSynced, + nodeInformer.Informer().HasSynced, + l2Informer.Informer().HasSynced, + } + for !cache.WaitForNamedCacheSync("l4", waitOrTimeout(ctx, time.Minute), synced...) { + if ctx.Err() != nil { + return nil + } + l4log.Info("still waiting for the L4 caches to sync; is the MetalLB ServiceL2Status CRD installed?") + } + + svcLister := svcInformer.Lister() + sliceLister := sliceInformer.Lister() + nodeLister := nodeInformer.Lister() + l2Lister := l2Informer.Lister() + c.input = func() l4.Input { + in := l4.Input{NodeName: c.NodeName, Announced: map[string]bool{}} + in.Services, _ = svcLister.List(labels.Everything()) + in.EndpointSlices, _ = sliceLister.List(labels.Everything()) + in.Nodes, _ = nodeLister.List(labels.Everything()) + objs, _ := l2Lister.List(labels.Everything()) + for _, o := range objs { + if key, ok := announcedService(o, c.NodeName); ok { + in.Announced[key] = true + } + } + return in + } + + c.run(ctx) + l4log.Info("stopping the L4 LoadBalancer mode; its table stays in place") + return nil +} + +// run is the sync loop. +func (c *L4Controller) run(ctx context.Context) { + resync := time.NewTicker(orDefault(c.ResyncInterval, defaultL4ResyncInterval)) + defer resync.Stop() + var retry <-chan time.Time + + force := true // the first pass replaces whatever the previous instance left + for { + if err := c.syncWith(c.input(), force); err != nil { + l4log.Error(err, "L4 sync failed, retrying") + retry = time.After(orDefault(c.RetryInterval, defaultL4RetryInterval)) + } else { + retry = nil + } + + force = false + select { + case <-ctx.Done(): + return + case <-c.dirty: + case <-retry: + case <-resync.C: + force = true + } + // Let a burst of events settle into one pass. + select { + case <-ctx.Done(): + return + case <-time.After(orDefault(c.MinSyncInterval, defaultL4MinSyncInterval)): + } + } +} + +func (c *L4Controller) markDirty() { + select { + case c.dirty <- struct{}{}: + default: + } +} + +// syncWith programs the state computed from in, then purges the conntrack +// entries it left stale. An unchanged state is not reprogrammed unless force +// is set. +func (c *L4Controller) syncWith(in l4.Input, force bool) error { + st, notices := l4.Build(in) + c.logNotices(notices) + + unchanged := c.applied != nil && reflect.DeepEqual(*c.applied, st) + if !unchanged || force { + if err := c.Datapath.Sync(st); err != nil { + return fmt.Errorf("programming the L4 table: %w", err) + } + if !unchanged { + l4log.Info("L4 table programmed", "vips", len(st.VIPs), "ports", len(st.Ports), "translated", len(st.Rules)) + } + prev := c.applied + if c.purgePending || l4.PurgeNeeded(prev, st) { + c.purgePending = true + if prev != nil { + if c.purgeVIPs == nil { + c.purgeVIPs = map[netip.Addr]struct{}{} + } + for _, v := range prev.VIPs { + c.purgeVIPs[v] = struct{}{} + } + } + } + c.applied = &st + } + + if !c.purgePending { + return nil + } + // The previous VIPs keep the flows of a service removed since the last + // successful purge within reach. + before := &l4.State{} + for v := range c.purgeVIPs { + before.VIPs = append(before.VIPs, v) + } + n, err := c.Conntrack.Purge(l4.StaleFlows(before, st)) + if err != nil { + return fmt.Errorf("purging stale L4 conntrack entries: %w", err) + } + if n > 0 { + l4log.Info("purged stale L4 conntrack entries", "count", n) + } + c.purgePending = false + c.purgeVIPs = nil + return nil +} + +func (c *L4Controller) logNotices(notices []l4.Notice) { + current := map[string]string{} + for _, n := range notices { + if prev, ok := current[n.Service]; ok { + current[n.Service] = prev + "; " + n.Message + } else { + current[n.Service] = n.Message + } + } + for svc, msg := range current { + if c.notices[svc] != msg { + l4log.Info("service not fully programmed", "service", svc, "reason", msg) + } + } + c.notices = current +} + +// announcedService returns the service a ServiceL2Status says node announces. +func announcedService(obj interface{}, node string) (string, bool) { + u, ok := obj.(*unstructured.Unstructured) + if !ok { + return "", false + } + // The informer already filters on the node label; the status is the + // authoritative field. + if n, _, _ := unstructured.NestedString(u.Object, "status", "node"); n != node { + return "", false + } + ns, _, _ := unstructured.NestedString(u.Object, "status", "serviceNamespace") + name, _, _ := unstructured.NestedString(u.Object, "status", "serviceName") + if ns == "" || name == "" { + return "", false + } + return ns + "/" + name, true +} + +// waitOrTimeout returns a channel closed when ctx ends or after d, whichever +// comes first, so a cache sync that never completes is reported periodically. +func waitOrTimeout(ctx context.Context, d time.Duration) <-chan struct{} { + ch := make(chan struct{}) + go func() { + defer close(ch) + select { + case <-ctx.Done(): + case <-time.After(d): + } + }() + return ch +} + +func orDefault(d, def time.Duration) time.Duration { + if d <= 0 { + return def + } + return d +} diff --git a/pkg/controllers/l4_controller_test.go b/pkg/controllers/l4_controller_test.go new file mode 100644 index 0000000..a194831 --- /dev/null +++ b/pkg/controllers/l4_controller_test.go @@ -0,0 +1,376 @@ +package controllers + +import ( + "context" + "errors" + "net/netip" + "reflect" + "sync" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + + "github.com/cozystack/cozy-proxy/pkg/l4" +) + +// fakeL4Datapath records the states it is asked to program. +type fakeL4Datapath struct { + mu sync.Mutex + synced []l4.State + failing bool +} + +func (f *fakeL4Datapath) Sync(st l4.State) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.failing { + return errors.New("commit refused") + } + f.synced = append(f.synced, st) + return nil +} + +func (f *fakeL4Datapath) Teardown() error { return nil } + +func (f *fakeL4Datapath) count() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.synced) +} + +func (f *fakeL4Datapath) last() (l4.State, bool) { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.synced) == 0 { + return l4.State{}, false + } + return f.synced[len(f.synced)-1], true +} + +// fakePurger runs the predicate over a fixed set of flows, as the conntrack +// table would hold them. +type fakePurger struct { + flows []l4.Flow + calls int + purged []l4.Flow + failing bool +} + +func (f *fakePurger) Purge(stale func(l4.Flow) bool) (uint, error) { + f.calls++ + if f.failing { + return 0, errors.New("dump interrupted") + } + var n uint + var left []l4.Flow + for _, fl := range f.flows { + if stale(fl) { + f.purged = append(f.purged, fl) + n++ + continue + } + left = append(left, fl) + } + f.flows = left + return n, nil +} + +const l4Node = "node-a" + +func l4Svc(name, vip string) *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: name, + Labels: map[string]string{l4.ProxyLabel: l4.ProxyLabelValue}, + Annotations: map[string]string{l4.CiliumTypeAnnotation: l4.CiliumTypeClusterIP}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, + Ports: []v1.ServicePort{{Name: "pg", Port: 5432, Protocol: v1.ProtocolTCP}}, + }, + Status: v1.ServiceStatus{LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{{IP: vip}}, + }}, + } +} + +func l4Slice(svc string, ips ...string) *discoveryv1.EndpointSlice { + name, port, proto, ready, node := "pg", int32(5432), v1.ProtocolTCP, true, l4Node + s := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: svc + "-1", + Labels: map[string]string{discoveryv1.LabelServiceName: svc}, + }, + AddressType: discoveryv1.AddressTypeIPv4, + Ports: []discoveryv1.EndpointPort{{Name: &name, Port: &port, Protocol: &proto}}, + } + for _, ip := range ips { + s.Endpoints = append(s.Endpoints, discoveryv1.Endpoint{ + Addresses: []string{ip}, + Conditions: discoveryv1.EndpointConditions{Ready: &ready}, + NodeName: &node, + }) + } + return s +} + +func l4Input(slices ...*discoveryv1.EndpointSlice) l4.Input { + return l4.Input{ + NodeName: l4Node, + Services: []*v1.Service{l4Svc("pg", "192.0.2.10")}, + EndpointSlices: slices, + Announced: map[string]bool{"ns/pg": true}, + } +} + +func flowTo(backend string) l4.Flow { + return l4.Flow{ + Protocol: 6, + OrigDst: netip.MustParseAddrPort("192.0.2.10:5432"), + ReplySrc: netip.MustParseAddrPort(backend), + } +} + +// The first sync programs the state Build computes and purges once, since the +// previous instance may have left translations this one no longer programs. +func TestL4SyncProgramsAndPurgesOnFirstSync(t *testing.T) { + dp, ct := &fakeL4Datapath{}, &fakePurger{flows: []l4.Flow{flowTo("10.0.0.1:5432"), flowTo("10.0.0.9:5432")}} + c := &L4Controller{NodeName: l4Node, Datapath: dp, Conntrack: ct} + + in := l4Input(l4Slice("pg", "10.0.0.1")) + if err := c.syncWith(in, false); err != nil { + t.Fatalf("sync: %v", err) + } + + want, _ := l4.Build(in) + if got, ok := dp.last(); !ok || !reflect.DeepEqual(got, want) { + t.Errorf("programmed %+v, want %+v", got, want) + } + if ct.calls != 1 { + t.Fatalf("the first sync must purge once, got %d", ct.calls) + } + if len(ct.purged) != 1 || ct.purged[0] != flowTo("10.0.0.9:5432") { + t.Errorf("purged %+v, want only the flow to the backend no longer programmed", ct.purged) + } +} + +// An unchanged state is neither reprogrammed nor purged again. +func TestL4SyncSkipsUnchangedState(t *testing.T) { + dp, ct := &fakeL4Datapath{}, &fakePurger{} + c := &L4Controller{NodeName: l4Node, Datapath: dp, Conntrack: ct} + in := l4Input(l4Slice("pg", "10.0.0.1")) + + for i := 0; i < 3; i++ { + if err := c.syncWith(in, false); err != nil { + t.Fatalf("sync %d: %v", i, err) + } + } + if dp.count() != 1 || ct.calls != 1 { + t.Errorf("an unchanged state must be programmed once, got %d syncs and %d purges", dp.count(), ct.calls) + } + + // The periodic resync re-applies it anyway, so a table removed by hand + // comes back, but has nothing to purge. + if err := c.syncWith(in, true); err != nil { + t.Fatalf("forced sync: %v", err) + } + if dp.count() != 2 || ct.calls != 1 { + t.Errorf("a forced sync must reprogram without purging, got %d syncs and %d purges", dp.count(), ct.calls) + } +} + +func TestL4SyncPurgesWhenABackendIsWithdrawn(t *testing.T) { + dp, ct := &fakeL4Datapath{}, &fakePurger{} + c := &L4Controller{NodeName: l4Node, Datapath: dp, Conntrack: ct} + + if err := c.syncWith(l4Input(l4Slice("pg", "10.0.0.1", "10.0.0.2")), false); err != nil { + t.Fatalf("sync: %v", err) + } + ct.flows = []l4.Flow{flowTo("10.0.0.1:5432"), flowTo("10.0.0.2:5432")} + + // A backend added: nothing to purge. + if err := c.syncWith(l4Input(l4Slice("pg", "10.0.0.1", "10.0.0.2", "10.0.0.3")), false); err != nil { + t.Fatalf("sync: %v", err) + } + if ct.calls != 1 { + t.Errorf("adding a backend must not purge, got %d purges", ct.calls) + } + + // A backend withdrawn: its flows go. + if err := c.syncWith(l4Input(l4Slice("pg", "10.0.0.1", "10.0.0.3")), false); err != nil { + t.Fatalf("sync: %v", err) + } + if ct.calls != 2 || len(ct.purged) != 1 || ct.purged[0] != flowTo("10.0.0.2:5432") { + t.Errorf("withdrawing a backend must purge its flows, got %d purges, purged %+v", ct.calls, ct.purged) + } +} + +// A refused commit leaves the applied state as it was, and the next attempt +// commits even though the desired state did not change in between. +func TestL4SyncRetriesAFailedCommit(t *testing.T) { + dp, ct := &fakeL4Datapath{failing: true}, &fakePurger{} + c := &L4Controller{NodeName: l4Node, Datapath: dp, Conntrack: ct} + in := l4Input(l4Slice("pg", "10.0.0.1")) + + if err := c.syncWith(in, false); err == nil { + t.Fatal("a refused commit must be reported") + } + if ct.calls != 0 { + t.Error("nothing may be purged before the commit succeeded") + } + + dp.failing = false + if err := c.syncWith(in, false); err != nil { + t.Fatalf("sync: %v", err) + } + if dp.count() != 1 || ct.calls != 1 { + t.Errorf("the retry must commit and purge, got %d syncs and %d purges", dp.count(), ct.calls) + } +} + +// A failed purge is retried on the next sync, and still covers the VIP of a +// service that was removed in the meantime. +func TestL4SyncRetriesAFailedPurge(t *testing.T) { + dp, ct := &fakeL4Datapath{}, &fakePurger{} + c := &L4Controller{NodeName: l4Node, Datapath: dp, Conntrack: ct} + + if err := c.syncWith(l4Input(l4Slice("pg", "10.0.0.1")), false); err != nil { + t.Fatalf("sync: %v", err) + } + + // The service goes away, and the purge that follows fails. + ct.flows = []l4.Flow{flowTo("10.0.0.1:5432")} + ct.failing = true + gone := l4.Input{NodeName: l4Node} + if err := c.syncWith(gone, false); err == nil { + t.Fatal("a failed purge must be reported") + } + + ct.failing = false + if err := c.syncWith(gone, false); err != nil { + t.Fatalf("sync: %v", err) + } + if len(ct.purged) != 1 { + t.Errorf("the retried purge must still reach the removed service's flows, purged %+v", ct.purged) + } + if err := c.syncWith(gone, false); err != nil { + t.Fatalf("sync: %v", err) + } + // The first sync, the failed one and its retry; the last sync has + // nothing left to do. + if ct.calls != 3 { + t.Errorf("purges = %d, want 3", ct.calls) + } +} + +var l2StatusGVR = schema.GroupVersionResource{Group: "metallb.io", Version: "v1beta1", Resource: "servicel2statuses"} + +func l2Status(name, node, svcNS, svcName string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("metallb.io/v1beta1") + u.SetKind("ServiceL2Status") + u.SetNamespace("metallb-system") + u.SetName(name) + u.SetLabels(map[string]string{"metallb.io/node": node}) + _ = unstructured.SetNestedField(u.Object, node, "status", "node") + _ = unstructured.SetNestedField(u.Object, svcName, "status", "serviceName") + _ = unstructured.SetNestedField(u.Object, svcNS, "status", "serviceNamespace") + return u +} + +// Wiring: the controller reads Services, EndpointSlices, Nodes and MetalLB's +// ServiceL2Status from the API, and programs what Build makes of them. +func TestL4ControllerProgramsFromTheAPI(t *testing.T) { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: l4Node}, + Status: v1.NodeStatus{Addresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.200.24.11"}}}, + } + cs := fake.NewSimpleClientset(l4Svc("pg", "192.0.2.10"), l4Slice("pg", "10.0.0.1"), node) + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), + map[schema.GroupVersionResource]string{l2StatusGVR: "ServiceL2StatusList"}, + // Another node's status must not make this node an announcer. + l2Status("l2-other", "node-b", "ns", "pg"), + ) + + dp := &fakeL4Datapath{} + c := &L4Controller{ + Clientset: cs, + Dynamic: dyn, + NodeName: l4Node, + Datapath: dp, + Conntrack: &fakePurger{}, + MinSyncInterval: time.Millisecond, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- c.Start(ctx) }() + defer func() { + cancel() + if err := <-done; err != nil { + t.Errorf("Start returned %v", err) + } + }() + + notAnnounced := func(st l4.State) bool { + return len(st.VIPs) == 1 && len(st.Ports) == 1 && len(st.Rules) == 0 && len(st.NodeIPs) == 1 + } + waitFor(t, dp, "the VIP guarded but not translated", notAnnounced) + + // MetalLB elects this node. + if _, err := dyn.Resource(l2StatusGVR).Namespace("metallb-system").Create(ctx, + l2Status("l2-here", l4Node, "ns", "pg"), metav1.CreateOptions{}); err != nil { + t.Fatalf("creating the ServiceL2Status: %v", err) + } + waitFor(t, dp, "the VIP translated to the local backend", func(st l4.State) bool { + return len(st.Rules) == 1 && len(st.Rules[0].Backends) == 1 && + st.Rules[0].Backends[0].IP == netip.MustParseAddr("10.0.0.1") + }) + + // The backend goes away. + if _, err := cs.DiscoveryV1().EndpointSlices("ns").Update(ctx, l4Slice("pg"), metav1.UpdateOptions{}); err != nil { + t.Fatalf("updating the EndpointSlice: %v", err) + } + waitFor(t, dp, "the port dropped for lack of backend", func(st l4.State) bool { + return len(st.Rules) == 1 && len(st.Rules[0].Backends) == 0 + }) +} + +func waitFor(t *testing.T, dp *fakeL4Datapath, what string, ok func(l4.State) bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if st, synced := dp.last(); synced && ok(st) { + return + } + time.Sleep(10 * time.Millisecond) + } + st, _ := dp.last() + t.Fatalf("timed out waiting for %s; last programmed state: %+v", what, st) +} + +// Without a node name the mode cannot tell what is local: it must refuse to +// program anything, and must not take the process down with it — the VM mode +// runs in the same manager. +func TestL4ControllerWithoutNodeNameProgramsNothing(t *testing.T) { + dp := &fakeL4Datapath{} + c := &L4Controller{Datapath: dp, Conntrack: &fakePurger{}} + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if err := c.Start(ctx); err != nil { + t.Errorf("Start must not fail the manager, got %v", err) + } + if dp.count() != 0 { + t.Errorf("nothing may be programmed without a node name, got %d syncs", dp.count()) + } +} From 6576eab13216c833f954ac5a4e766284772d4f08 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:27:29 +0200 Subject: [PATCH 20/42] main: run the L4 mode behind --enable-l4-loadbalancer The mode is off by default, so an upgrade changes nothing for the VM mode. When it is off, a table left by an earlier run is removed at startup: that is how the whole mode is rolled back. It needs NODE_NAME, and is not started without it rather than programming every node. Its clients are built from an untouched copy of the manager's config, since the VM mode pins the shared one to the core group. No failure on the L4 side stops the process. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- main.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/main.go b/main.go index e870daa..59248bb 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth" "github.com/cozystack/cozy-proxy/pkg/controllers" + "github.com/cozystack/cozy-proxy/pkg/l4" "github.com/cozystack/cozy-proxy/pkg/proxy" "k8s.io/apimachinery/pkg/runtime" @@ -19,7 +20,9 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) var ( @@ -34,8 +37,12 @@ func init() { func main() { var probeAddr string var metricsAddr string + var enableL4 bool flag.StringVar(&probeAddr, "health-probe-bind-address", "0", "The address the probe endpoint binds to. Set to \"0\" to disable.") flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. Set to \"0\" to disable.") + flag.BoolVar(&enableL4, "enable-l4-loadbalancer", false, + "Run the L4 LoadBalancer mode for services labelled "+l4.ProxyLabel+"="+l4.ProxyLabelValue+ + " (see docs/rfc/l4-loadbalancer-mode.md). When false, a table left by an earlier run is removed.") opts := zap.Options{ Development: true, } @@ -54,6 +61,10 @@ func main() { os.Exit(1) } + // The L4 mode builds its own clients from an untouched copy: the VM mode + // pins the shared config to the core group below. + l4Cfg := rest.CopyConfig(mgr.GetConfig()) + cfg := mgr.GetConfig() cfg.GroupVersion = &corev1.SchemeGroupVersion cfg.APIPath = "/api" @@ -89,6 +100,8 @@ func main() { os.Exit(1) } + setupL4(mgr, l4Cfg, enableL4, nodeName) + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { log.Error(err, "unable to set up health check") os.Exit(1) @@ -104,3 +117,42 @@ func main() { os.Exit(1) } } + +// setupL4 adds the L4 LoadBalancer mode to the manager, or removes what an +// earlier run of it left when it is disabled. Nothing here may stop the +// process: the VM mode runs in the same manager. +func setupL4(mgr ctrl.Manager, cfg *rest.Config, enabled bool, nodeName string) { + datapath := &proxy.NFTL4Datapath{} + if !enabled { + if err := datapath.Teardown(); err != nil { + log.Error(err, "could not remove the L4 table left by an earlier run") + } + return + } + if nodeName == "" { + log.Error(nil, "the L4 LoadBalancer mode needs NODE_NAME to tell which backends are local; not starting it") + return + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + log.Error(err, "failed to create the L4 clientset; not starting the L4 mode") + return + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + log.Error(err, "failed to create the L4 dynamic client; not starting the L4 mode") + return + } + if err := mgr.Add(&controllers.L4Controller{ + Clientset: clientset, + Dynamic: dyn, + NodeName: nodeName, + Datapath: datapath, + Conntrack: &proxy.NetlinkConntrack{}, + }); err != nil { + log.Error(err, "unable to add the L4 controller to the manager") + return + } + log.Info("L4 LoadBalancer mode enabled") +} From ccc5db045a89eb3277c54f2a87f3dd096f085451 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:27:29 +0200 Subject: [PATCH 21/42] chart: add an l4LoadBalancer.enabled switch Off by default, and the rendered chart is then unchanged. On, it passes --enable-l4-loadbalancer and grants what the mode watches: Nodes, EndpointSlices and MetalLB's ServiceL2Status, read only. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- charts/cozy-proxy/templates/daemonset.yaml | 4 ++++ charts/cozy-proxy/templates/role.yaml | 11 +++++++++++ charts/cozy-proxy/values.yaml | 7 +++++++ 3 files changed, 22 insertions(+) diff --git a/charts/cozy-proxy/templates/daemonset.yaml b/charts/cozy-proxy/templates/daemonset.yaml index c06bdfa..a66eb62 100644 --- a/charts/cozy-proxy/templates/daemonset.yaml +++ b/charts/cozy-proxy/templates/daemonset.yaml @@ -21,6 +21,10 @@ spec: - name: cozy-proxy image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.l4LoadBalancer.enabled }} + args: + - --enable-l4-loadbalancer + {{- end }} env: # Scopes the ingress rules to the backends hosted on this node; see # "Rule scoping" in the README for why a non-hosting node must not diff --git a/charts/cozy-proxy/templates/role.yaml b/charts/cozy-proxy/templates/role.yaml index 5e80fe1..f01596d 100644 --- a/charts/cozy-proxy/templates/role.yaml +++ b/charts/cozy-proxy/templates/role.yaml @@ -9,4 +9,15 @@ rules: - apiGroups: [""] resources: ["services", "endpoints"] verbs: ["get", "list", "watch"] + {{- if .Values.l4LoadBalancer.enabled }} + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list", "watch"] + - apiGroups: ["metallb.io"] + resources: ["servicel2statuses"] + verbs: ["list", "watch"] + {{- end }} {{- end }} diff --git a/charts/cozy-proxy/values.yaml b/charts/cozy-proxy/values.yaml index e143e92..f749c34 100644 --- a/charts/cozy-proxy/values.yaml +++ b/charts/cozy-proxy/values.yaml @@ -10,3 +10,10 @@ daemonset: rbac: create: true + +# L4 LoadBalancer mode: takes over from Cilium the public IP of services +# labelled networking.cozystack.io/lb-proxy=cozy-proxy that also carry +# service.cilium.io/type=ClusterIP. See docs/rfc/l4-loadbalancer-mode.md. +# Disabling it removes the table it programmed on the next start. +l4LoadBalancer: + enabled: false From 49ce09796b9608c8fe88fdaf4309f3e460443b78 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:28:26 +0200 Subject: [PATCH 22/42] proxy: check that rebuilding the L4 table keeps connections alive Every sync deletes and recreates the table, and a restarted instance does the same on its first pass. That is only safe because a flow's translation lives in conntrack rather than in the rules; this test holds a connection through the VIP open across three rebuilds. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/proxy/l4_datapath_test.go | 79 +++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/pkg/proxy/l4_datapath_test.go b/pkg/proxy/l4_datapath_test.go index 70c745c..56de149 100644 --- a/pkg/proxy/l4_datapath_test.go +++ b/pkg/proxy/l4_datapath_test.go @@ -318,3 +318,82 @@ func isTimeout(err error) bool { var ne net.Error return errors.As(err, &ne) && ne.Timeout() } + +// echo serves a line-echo on port 9090 of the backends, keeping each +// connection open until the client closes it. +func echo(t *testing.T, ns netns.NsHandle) { + t.Helper() + var ln net.Listener + must(t, "listen", doIn(t, ns, func() error { + var err error + ln, err = net.Listen("tcp4", "0.0.0.0:9090") + return err + })) + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + r := bufio.NewReader(c) + for { + line, err := r.ReadString('\n') + if err != nil { + return + } + if _, err := c.Write([]byte(line)); err != nil { + return + } + } + }() + } + }() +} + +// Every sync rebuilds the table, and a restarted instance does the same on its +// first pass. A connection established before must keep flowing: its +// translation lives in conntrack, not in the rules. +func TestL4DatapathResyncKeepsEstablishedConnections(t *testing.T) { + tp := newTopology(t) + echo(t, tp.server) + dp := datapathIn(tp.router) + st := l4.State{ + VIPs: []netip.Addr{a(testVIP)}, + Ports: []l4.PortKey{pk(testVIP, v1.ProtocolTCP, 90)}, + Rules: []l4.Rule{{ + PortKey: pk(testVIP, v1.ProtocolTCP, 90), + Service: "ns/echo", + Backends: []l4.Backend{bk(backendA, 9090)}, + }}, + } + must(t, "Sync", dp.Sync(st)) + + var conn net.Conn + must(t, "dial", doIn(t, tp.client, func() error { + var err error + conn, err = net.DialTimeout("tcp4", testVIP+":90", dialTimeout) + return err + })) + defer conn.Close() + r := bufio.NewReader(conn) + roundTrip := func(msg string) { + t.Helper() + _ = conn.SetDeadline(time.Now().Add(serverTimeout)) + if _, err := conn.Write([]byte(msg + "\n")); err != nil { + t.Fatalf("write %q: %v", msg, err) + } + got, err := r.ReadString('\n') + if err != nil || got != msg+"\n" { + t.Fatalf("echo of %q = %q, %v", msg, got, err) + } + } + + roundTrip("before") + for i := 0; i < 3; i++ { + must(t, "re-Sync", dp.Sync(st)) + } + roundTrip("after") +} From d32b1c6ca30c61ef00131cd4c6c50772be4a8001 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:29:21 +0200 Subject: [PATCH 23/42] proxy: check the Close errors in the L4 tests golangci-lint's errcheck flagged the deferred Close calls added with the L4 tests. The branch now adds no finding to the 31 the default linters already report on the VM mode's code. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/proxy/l4_datapath_test.go | 8 ++++---- pkg/proxy/l4_nft_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/proxy/l4_datapath_test.go b/pkg/proxy/l4_datapath_test.go index 56de149..2d952b9 100644 --- a/pkg/proxy/l4_datapath_test.go +++ b/pkg/proxy/l4_datapath_test.go @@ -54,7 +54,7 @@ func doIn(t *testing.T, ns netns.NsHandle, f func() error) error { if err != nil { t.Fatalf("netns.Get: %v", err) } - defer orig.Close() + defer func() { _ = orig.Close() }() if err := netns.Set(ns); err != nil { t.Fatalf("netns.Set: %v", err) } @@ -167,7 +167,7 @@ func dial(t *testing.T, ns netns.NsHandle, addr string) (answer, error) { if err != nil { return answer{}, err } - defer conn.Close() + defer func() { _ = conn.Close() }() _ = conn.SetReadDeadline(time.Now().Add(serverTimeout)) line, err := bufio.NewReader(conn).ReadString('\n') if err != nil { @@ -337,7 +337,7 @@ func echo(t *testing.T, ns netns.NsHandle) { return } go func() { - defer c.Close() + defer func() { _ = c.Close() }() r := bufio.NewReader(c) for { line, err := r.ReadString('\n') @@ -377,7 +377,7 @@ func TestL4DatapathResyncKeepsEstablishedConnections(t *testing.T) { conn, err = net.DialTimeout("tcp4", testVIP+":90", dialTimeout) return err })) - defer conn.Close() + defer func() { _ = conn.Close() }() r := bufio.NewReader(conn) roundTrip := func(msg string) { t.Helper() diff --git a/pkg/proxy/l4_nft_test.go b/pkg/proxy/l4_nft_test.go index a14bbc7..72ec36d 100644 --- a/pkg/proxy/l4_nft_test.go +++ b/pkg/proxy/l4_nft_test.go @@ -43,7 +43,7 @@ func scratchNetns(t *testing.T) netns.NsHandle { if err != nil { t.Fatalf("netns.Get: %v", err) } - defer orig.Close() + defer func() { _ = orig.Close() }() ns, err := netns.New() // also switches this thread into it if err != nil { t.Skipf("cannot create a network namespace: %v", err) @@ -69,7 +69,7 @@ func inNetns(t *testing.T, ns netns.NsHandle, name string, args ...string) (stri if err != nil { t.Fatalf("netns.Get: %v", err) } - defer orig.Close() + defer func() { _ = orig.Close() }() if err := netns.Set(ns); err != nil { t.Fatalf("netns.Set: %v", err) } From c2f715832ee3db3dbf9073244faa6ed5835ef4a3 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:30:00 +0200 Subject: [PATCH 24/42] docs: document the L4 mode and align the RFC with the prototype MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README gets a short section on how a service opts in and how the mode is switched on. The RFC now describes the masquerade as the prototype programs it — matched on the translated destination, since the original-destination match nft would write makes "nft list" abort when emitted through github.com/google/nftables — the sync intervals, the flag, and what the prototype covers and how it is tested. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 28 +++++++++++++ docs/rfc/l4-loadbalancer-mode.md | 69 ++++++++++++++++++++++++++------ 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 0360a86..b180caf 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,34 @@ watches every Service, but programs: everything is programmed everywhere, so the binary still runs under a deployment that does not inject it. +## L4 LoadBalancer mode (experimental) + +Besides VMs, cozy-proxy can take over the public IP of ordinary `LoadBalancer` +services — several ports, several backends, port translation — so that it is +reachable from every pod the way it is from the Internet. A service opts in with +both: + +```yaml +metadata: + labels: + networking.cozystack.io/lb-proxy: cozy-proxy + annotations: + service.cilium.io/type: ClusterIP # Cilium releases the VIP, keeps the ClusterIP +spec: + type: LoadBalancer + externalTrafficPolicy: Local +``` + +The node MetalLB (L2) announces the VIP from DNATs new connections to its local +ready backends, masquerading only sources that are node IPs; everything else +addressed to the VIP is dropped. The mode has its own table, `ip +cozy_proxy_l4`, and never touches the VM mode's. It is off by default: run with +`--enable-l4-loadbalancer` (chart value `l4LoadBalancer.enabled`). Phase 1 +handles TCP, IPv4 and `externalTrafficPolicy: Local`. + +See [the RFC](docs/rfc/l4-loadbalancer-mode.md) for the model, the datapath, +the security implications and the rollout plan. + ## Installation Install controller using Helm-chart: diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index 0f1113c..920b1c7 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -2,7 +2,7 @@ | | | |---|---| -| Status | Draft, prototype on branch `feat/l4-loadbalancer` | +| Status | Draft. Phase 1 prototype on branch `feat/l4-loadbalancer` (section 11) | | Scope | cozy-proxy, plus the Cozystack charts that would opt services in | | Supersedes | nothing; the existing VM mode is unchanged | @@ -189,6 +189,8 @@ table ip cozy_proxy_l4 { set vip_ports { type ipv4_addr . inet_proto . inet_service } # InternalIP of every Node. set node_ips { type ipv4_addr } + # (backend IP, protocol, backend port) of every local target, announcer only. + set backends { type ipv4_addr . inet_proto . inet_service } # One map per announced service port: round-robin slot -> backend. map backends-//tcp/ { type integer : ipv4_addr . inet_service } @@ -208,7 +210,7 @@ table ip cozy_proxy_l4 { chain masq { type nat hook postrouting priority srcnat - 5; policy accept; - ct status dnat ip saddr @node_ips ct original ip daddr @vips masquerade + ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade } } ``` @@ -221,9 +223,18 @@ Notes: port) are dropped, which includes ICMP echo: nobody would answer it anyway. - `dnat` only holds rules on the announcer. Elsewhere the packet is left alone and routed out, to the announcer. -- `masq` only matches flows this node translated (`ct status dnat`) towards a - VIP, from a node IP. The masquerade picks the address of the interface towards - the backend, `ovn0` on kube-ovn, hence the `100.64.0.x` source. +- `masq` only matches flows this node translated (`ct status dnat`) towards one + of its L4 targets, from a node IP, so a node process talking to a backend pod + directly (a kubelet probe) keeps its source. The masquerade picks the address + of the interface towards the backend, `ovn0` on kube-ovn, hence the + `100.64.0.x` source. nft itself would match `ct original ip daddr @vips`; + `github.com/google/nftables` can only emit that with the generic conntrack + key, which makes `nft list` abort on the whole ruleset, hence the match on the + translated destination. +- `nft list` shows the backend maps as `type 0 : ipv4_addr . inet_service`: nft + only names an integer key through a `typeof` annotation, which + `github.com/google/nftables` cannot write. The elements are listed correctly + and the kernel does not look at either. - `numgen inc` is per node and per rule, which is enough: only one node translates a given VIP. @@ -290,10 +301,10 @@ it the mode refuses to start rather than program every node. ### 4.2 Reconciliation Level-triggered. Every informer event only marks the state dirty. A single sync -loop, rate-limited, recomputes the desired state from the informer caches and -applies it as one transaction (3.3). A failed sync stays dirty and is retried -on the next tick; a periodic resync re-applies the state even without events, so -a table deleted by hand comes back. +loop recomputes the desired state from the informer caches, coalescing bursts +(250 ms), and applies it as one transaction (3.3) when it changed. A failed pass +is retried after 5 s; a periodic resync (1 min) re-applies the state even +without events, so a table deleted or altered by hand comes back. The first sync waits for every informer to be synced. Until then the table left by the previous instance stays in place and keeps forwarding. @@ -324,8 +335,12 @@ commit, so no new flow can be created towards a backend that was just purged. update therefore does not interrupt traffic: the old rules keep forwarding until the new instance's first sync replaces them atomically. Changes that happen during the gap are applied with that delay, as with kube-proxy. -- Disabling the mode (flag off) deletes the table at startup, which is the - rollback path for the whole mode. +- The mode is behind `--enable-l4-loadbalancer` (chart: + `l4LoadBalancer.enabled`), off by default. Disabling it deletes the table at + startup, which is the rollback path for the whole mode. +- Nothing on the L4 side stops the process: the VM mode runs in the same + manager. Without `NODE_NAME`, or while the MetalLB CRD is missing, the mode + logs and stays idle. ### 4.5 Announcer changes @@ -459,6 +474,36 @@ another tenant, the source IP logged by the backend, MetalLB announcer, and the traffic for the PROXY-protocol host ingress. Its interaction with a VIP that Cilium no longer owns has to be checked before step 3. 9. **Observability**: metrics (sync duration and errors, programmed services, - purged flows) and Events on the Service when it is refused. + purged flows) and Events on the Service when it is refused. Phase 1 only + logs, once per change. 10. **Where the label lives**: `networking.cozystack.io/lb-proxy: cozy-proxy` is a proposal; the maintainers may prefer another key. +11. **`google/nftables` limits**: no `typeof` on sets, and no direction on the + typed conntrack keys (3.1). Both are cosmetic for `nft list` today; fixing + them upstream would let the masquerade match the original destination. + +## 11. Prototype status + +Branch `feat/l4-loadbalancer`, phase 1 only: + +| Area | Where | Tests | +|---|---|---| +| Selection (2.5) | `pkg/l4/select.go` | unit | +| Desired state per node (4.1) | `pkg/l4/state.go` | unit | +| Purge decision (4.3) | `pkg/l4/conntrack.go` | unit | +| nftables table (3) | `pkg/proxy/l4_nft.go` | kernel, in a network namespace: `nft list` against golden files, and real TCP traffic through three namespaces (translation, round-robin, source kept or masqueraded, guard, port without backend, purge, rebuild with an open connection) | +| Conntrack purge | `pkg/proxy/conntrack.go` | kernel, in a network namespace | +| Controller (4.2) | `pkg/controllers/l4_controller.go` | unit, and fake API clients | +| Switch and RBAC | `main.go`, chart `l4LoadBalancer.enabled` | `helm template` | + +The kernel tests need root and skip otherwise; `nft list` comparisons also need +the `nft` binary. They run in a privileged Linux container, for instance: + +``` +docker run --rm --privileged -v "$PWD":/src -w /src golang:1.26 sh -c \ + 'apt-get update -qq && apt-get install -y -qq nftables && go test ./...' +``` + +Not in phase 1: UDP, eTP `Cluster`, IPv6, terminating endpoints, metrics and +Events, and a lab validation of the binary itself (the model was validated with +hand-written rules). From b52e84f20a1b64a38d129235bdc26c91360a283a Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:30:33 +0200 Subject: [PATCH 25/42] l4: report each notice once A backend excluded because it belongs to a VM-mode service was reported once per port and per slice, and the controller logged the repetition joined together. Build now records a given message for a service only once. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/l4/state.go | 7 ++++++- pkg/l4/state_test.go | 12 ++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/l4/state.go b/pkg/l4/state.go index e4a17b7..f91d000 100644 --- a/pkg/l4/state.go +++ b/pkg/l4/state.go @@ -101,8 +101,13 @@ type builder struct { notices []Notice } +// notice records a message once, however many ports or slices raise it. func (b *builder) notice(service, format string, args ...any) { - b.notices = append(b.notices, Notice{Service: service, Message: fmt.Sprintf(format, args...)}) + n := Notice{Service: service, Message: fmt.Sprintf(format, args...)} + if slices.Contains(b.notices, n) { + return + } + b.notices = append(b.notices, n) } func (b *builder) addService(svc *v1.Service) { diff --git a/pkg/l4/state_test.go b/pkg/l4/state_test.go index d667e8b..e82b5a6 100644 --- a/pkg/l4/state_test.go +++ b/pkg/l4/state_test.go @@ -348,8 +348,8 @@ func TestBuildExcludesBackendsOwnedByTheVMMode(t *testing.T) { vm.Labels[vmProxyNameLabel] = vmProxyName web := l4Service("tenant", "web", "192.0.2.10") - web.Spec.Ports = []v1.ServicePort{svcPort("http", 80)} - ports := []discoveryv1.EndpointPort{epPort("http", 8080)} + web.Spec.Ports = []v1.ServicePort{svcPort("http", 80), svcPort("https", 443)} + ports := []discoveryv1.EndpointPort{epPort("http", 8080), epPort("https", 8443)} got, notices := Build(Input{ NodeName: thisNode, @@ -362,12 +362,16 @@ func TestBuildExcludesBackendsOwnedByTheVMMode(t *testing.T) { }) want := []Backend{be("10.0.0.8", 8080)} - if len(got.Rules) != 1 || !reflect.DeepEqual(got.Rules[0].Backends, want) { - t.Errorf("Rules = %+v, want only %v", got.Rules, want) + if len(got.Rules) != 2 || !reflect.DeepEqual(got.Rules[0].Backends, want) { + t.Errorf("Rules = %+v, want only %v on port 80", got.Rules, want) } if !hasNotice(notices, "tenant/web", "10.0.0.7") { t.Errorf("an excluded backend must be reported, got %v", notices) } + // Once, although both ports raise it. + if len(notices) != 1 { + t.Errorf("notices = %v, want a single one", notices) + } } // MetalLB can share a VIP between services on distinct ports. Two services From bed97a747043c1893cd52f490d44861888c16f90 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:39:24 +0200 Subject: [PATCH 26/42] main: allow running the L4 mode without the VM mode Trying the L4 mode on a live cluster should not mean replacing the cozy-proxy that serves the VMs. A second release with the L4 mode on must then keep its hands off the cozy_proxy table: the VM mode's startup rebuilds its chains and purges its maps, which would fight the instance that owns them. --enable-vm-mode=false (chart: vmMode.enabled=false) skips the VM controller. It defaults to true, so nothing changes otherwise, and the rendered chart stays the same. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- charts/cozy-proxy/templates/daemonset.yaml | 7 +++++- charts/cozy-proxy/values.yaml | 6 +++++ main.go | 26 +++++++++++++++------- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/charts/cozy-proxy/templates/daemonset.yaml b/charts/cozy-proxy/templates/daemonset.yaml index a66eb62..eea1af3 100644 --- a/charts/cozy-proxy/templates/daemonset.yaml +++ b/charts/cozy-proxy/templates/daemonset.yaml @@ -21,9 +21,14 @@ spec: - name: cozy-proxy image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if .Values.l4LoadBalancer.enabled }} + {{- if or .Values.l4LoadBalancer.enabled (not .Values.vmMode.enabled) }} args: + {{- if .Values.l4LoadBalancer.enabled }} - --enable-l4-loadbalancer + {{- end }} + {{- if not .Values.vmMode.enabled }} + - --enable-vm-mode=false + {{- end }} {{- end }} env: # Scopes the ingress rules to the backends hosted on this node; see diff --git a/charts/cozy-proxy/values.yaml b/charts/cozy-proxy/values.yaml index f749c34..16658d2 100644 --- a/charts/cozy-proxy/values.yaml +++ b/charts/cozy-proxy/values.yaml @@ -17,3 +17,9 @@ rbac: # Disabling it removes the table it programmed on the next start. l4LoadBalancer: enabled: false + +# VM mode, for services labelled service.kubernetes.io/service-proxy-name=cozy-proxy. +# Turn it off only to run the L4 mode as a second release next to a cozy-proxy +# that keeps the VM mode, for instance to try the L4 mode on a live cluster. +vmMode: + enabled: true diff --git a/main.go b/main.go index 59248bb..184caa0 100644 --- a/main.go +++ b/main.go @@ -38,11 +38,15 @@ func main() { var probeAddr string var metricsAddr string var enableL4 bool + var enableVM bool flag.StringVar(&probeAddr, "health-probe-bind-address", "0", "The address the probe endpoint binds to. Set to \"0\" to disable.") flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. Set to \"0\" to disable.") flag.BoolVar(&enableL4, "enable-l4-loadbalancer", false, "Run the L4 LoadBalancer mode for services labelled "+l4.ProxyLabel+"="+l4.ProxyLabelValue+ " (see docs/rfc/l4-loadbalancer-mode.md). When false, a table left by an earlier run is removed.") + flag.BoolVar(&enableVM, "enable-vm-mode", true, + "Run the VM mode for services labelled service.kubernetes.io/service-proxy-name=cozy-proxy. "+ + "Turn it off only for an instance running the L4 mode next to another cozy-proxy that keeps the VM mode.") opts := zap.Options{ Development: true, } @@ -89,15 +93,21 @@ func main() { "set it from spec.nodeName to scope rules to this node") } - controller := &controllers.ServicesController{ - Clientset: clientset, - Proxy: &proxy.NFTProxyProcessor{}, - NodeName: nodeName, - } + if enableVM { + controller := &controllers.ServicesController{ + Clientset: clientset, + Proxy: &proxy.NFTProxyProcessor{}, + NodeName: nodeName, + } - if err := mgr.Add(controller); err != nil { - log.Error(err, "unable to add endpoints controller to manager") - os.Exit(1) + if err := mgr.Add(controller); err != nil { + log.Error(err, "unable to add endpoints controller to manager") + os.Exit(1) + } + } else { + // Another instance owns the cozy_proxy table: leave it alone, the VM + // mode's startup would rebuild its chains and purge its maps. + log.Info("VM mode disabled, the cozy_proxy table is left untouched") } setupL4(mgr, l4Cfg, enableL4, nodeName) From fd5b8530376def57cc2a85a67d4439418dc29986 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:42:04 +0200 Subject: [PATCH 27/42] l4: let not-ready and terminating backends drain A backend that only stopped being ready, or started terminating, lost its live connections at the next purge: with loose TCP pickup the next packet of the flow was translated to another backend, which reset it. kube-proxy and Cilium let such connections finish, and an ingress-nginx rolling update relies on it. Such endpoints are now kept as draining: no new connection goes to them, but their flows are only purged once they leave the EndpointSlices, or the service, the port or the announcement goes. A terminating endpoint is also no longer taken as a backend when Ready still says true, which publishNotReadyAddresses makes it do. Found in review. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/rfc/l4-loadbalancer-mode.md | 10 ++++-- pkg/l4/conntrack.go | 11 +++++-- pkg/l4/conntrack_test.go | 22 +++++++++++++ pkg/l4/state.go | 55 +++++++++++++++++++++++--------- pkg/l4/state_test.go | 41 ++++++++++++++++++++++-- 5 files changed, 117 insertions(+), 22 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index 920b1c7..327b47f 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -322,8 +322,14 @@ the first sync, the mode deletes the conntrack entries that: destination), and - target a VIP the mode knows (previous or current state), and - point to a (VIP, protocol, port, backend IP, backend port) that is no longer - programmed **on this node** — backend gone or not ready, port removed, - service removed or no longer managed, or this node no longer the announcer. + programmed **on this node** — backend gone from the EndpointSlices, port + removed, service removed or no longer managed, or this node no longer the + announcer. + +A local endpoint that is still listed but not ready, or terminating, is +*draining*: it gets no new connection, but its live ones are kept until it +leaves the slices, so a backend shutting down gracefully (an ingress-nginx +rolling update) can finish them, as behind kube-proxy or Cilium. Untranslated flows towards a VIP (a pod on a non-announcer node passing through) are never touched, nor is anything the VM mode tracks. The purge runs after the diff --git a/pkg/l4/conntrack.go b/pkg/l4/conntrack.go index a5b031f..8835770 100644 --- a/pkg/l4/conntrack.go +++ b/pkg/l4/conntrack.go @@ -25,11 +25,16 @@ type target struct { backend netip.AddrPort } +// targets returns the translations whose flows must be kept: the ones the +// datapath performs, and the ones towards draining backends, which finish +// their connections. func targets(st State) map[target]struct{} { out := map[target]struct{}{} for _, r := range st.Rules { - for _, b := range r.Backends { - out[target{r.VIP, protocolNumber(r.Protocol), r.Port, netip.AddrPortFrom(b.IP, b.Port)}] = struct{}{} + for _, bs := range [][]Backend{r.Backends, r.Draining} { + for _, b := range bs { + out[target{r.VIP, protocolNumber(r.Protocol), r.Port, netip.AddrPortFrom(b.IP, b.Port)}] = struct{}{} + } } } return out @@ -44,7 +49,7 @@ func targets(st State) map[target]struct{} { // // An entry is stale when this node translated it (its replies do not come from // the address the client used), towards a VIP the L4 mode knows, and to a -// target the node no longer programs. Untranslated flows to a VIP — a pod on a +// target the node no longer programs nor drains. Untranslated flows to a VIP — a pod on a // node that does not announce it, passing through — are left alone, and so is // everything addressed elsewhere, which keeps the VM mode out of reach. func StaleFlows(prev *State, cur State) func(Flow) bool { diff --git a/pkg/l4/conntrack_test.go b/pkg/l4/conntrack_test.go index bac8184..9f738a0 100644 --- a/pkg/l4/conntrack_test.go +++ b/pkg/l4/conntrack_test.go @@ -118,3 +118,25 @@ func TestProtocolNumber(t *testing.T) { t.Error("IANA numbers expected for TCP and UDP") } } + +// A backend that stops being ready keeps its live connections until it leaves +// the EndpointSlice; only then are they purged. +func TestDrainingBackendKeepsItsFlows(t *testing.T) { + ready := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080), be("10.0.0.2", 8080))) + draining := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080))) + draining.Rules[0].Draining = []Backend{be("10.0.0.2", 8080)} + gone := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080))) + + if PurgeNeeded(&ready, draining) { + t.Error("a backend that starts draining must not trigger a purge") + } + if StaleFlows(&ready, draining)(translated(tcp, "192.0.2.10:80", "10.0.0.2:8080")) { + t.Error("a draining backend's flows must be kept") + } + if !PurgeNeeded(&draining, gone) { + t.Error("a draining backend that leaves the slice must trigger a purge") + } + if !StaleFlows(&draining, gone)(translated(tcp, "192.0.2.10:80", "10.0.0.2:8080")) { + t.Error("the flows of a backend gone from the slice must be purged") + } +} diff --git a/pkg/l4/state.go b/pkg/l4/state.go index f91d000..85dfa9e 100644 --- a/pkg/l4/state.go +++ b/pkg/l4/state.go @@ -35,6 +35,11 @@ type Rule struct { // Service is the namespace/name of the Service the frontend belongs to. Service string Backends []Backend + // Draining are local endpoints still listed but not ready, or + // terminating. They get no new connection, but their live ones are not + // purged: a backend shutting down gracefully must be able to finish them, + // as it would behind kube-proxy or Cilium. The datapath ignores them. + Draining []Backend } // State is everything one node programs for the L4 mode. Every slice is sorted, @@ -138,9 +143,9 @@ func (b *builder) addService(svc *v1.Service) { b.notice(key, "port %s/%d not programmed: only TCP is supported", proto, sp.Port) continue } - var backends []Backend + var backends, draining []Backend if announced { - backends = b.localBackends(key, svc.Namespace, svc.Name, sp.Name, proto) + backends, draining = b.localBackends(key, svc.Namespace, svc.Name, sp.Name, proto) } for _, vip := range vips { pk := PortKey{VIP: vip, Protocol: proto, Port: uint16(sp.Port)} @@ -151,17 +156,17 @@ func (b *builder) addService(svc *v1.Service) { b.owner[pk] = key b.st.Ports = append(b.st.Ports, pk) if announced { - b.st.Rules = append(b.st.Rules, Rule{PortKey: pk, Service: key, Backends: backends}) + b.st.Rules = append(b.st.Rules, Rule{PortKey: pk, Service: key, Backends: backends, Draining: draining}) } } } } -// localBackends returns the ready endpoints of the service port hosted on this -// node, sorted and deduplicated. The port is matched by name and protocol, the -// way kube-proxy does, which also resolves named target ports. -func (b *builder) localBackends(key, ns, name, portName string, proto v1.Protocol) []Backend { - var out []Backend +// localBackends returns the endpoints of the service port hosted on this node, +// sorted and deduplicated: the ones serving, and the draining ones. The port +// is matched by name and protocol, the way kube-proxy does, which also +// resolves named target ports. +func (b *builder) localBackends(key, ns, name, portName string, proto v1.Protocol) (ready, draining []Backend) { for _, s := range b.slices[ns+"/"+name] { if s.AddressType != discoveryv1.AddressTypeIPv4 { continue @@ -171,7 +176,7 @@ func (b *builder) localBackends(key, ns, name, portName string, proto v1.Protoco continue } for _, ep := range s.Endpoints { - if !ready(ep) || ep.NodeName == nil || *ep.NodeName != b.in.NodeName || len(ep.Addresses) == 0 { + if ep.NodeName == nil || *ep.NodeName != b.in.NodeName || len(ep.Addresses) == 0 { continue } ip, err := netip.ParseAddr(ep.Addresses[0]) @@ -182,12 +187,27 @@ func (b *builder) localBackends(key, ns, name, portName string, proto v1.Protoco b.notice(key, "backend %s excluded: it is the pod of VM-mode service %s", ip, owner) continue } - out = append(out, Backend{IP: ip, Port: port}) + if serves(ep) { + ready = append(ready, Backend{IP: ip, Port: port}) + } else { + draining = append(draining, Backend{IP: ip, Port: port}) + } } } - return sortedUnique(out, func(x, y Backend) int { - return cmp.Or(x.IP.Compare(y.IP), cmp.Compare(x.Port, y.Port)) + ready = sortedUnique(ready, compareBackend) + // An endpoint listed both ways during a transition counts as serving. + draining = slices.DeleteFunc(sortedUnique(draining, compareBackend), func(d Backend) bool { + _, found := slices.BinarySearchFunc(ready, d, compareBackend) + return found }) + if len(draining) == 0 { + draining = nil + } + return ready, draining +} + +func compareBackend(x, y Backend) int { + return cmp.Or(x.IP.Compare(y.IP), cmp.Compare(x.Port, y.Port)) } // vmModePods returns the pod IPs backing a VM-mode service. The VM mode @@ -212,9 +232,14 @@ func (b *builder) vmModePods() map[netip.Addr]string { return out } -// ready follows the EndpointSlice API: a nil Ready is to be read as ready. -// A terminating endpoint is not ready. -func ready(ep discoveryv1.Endpoint) bool { +// serves reports whether an endpoint may take new connections. A nil Ready is +// to be read as ready, per the EndpointSlice API. A terminating endpoint does +// not serve even when Ready says true, which publishNotReadyAddresses makes it +// do. +func serves(ep discoveryv1.Endpoint) bool { + if ep.Conditions.Terminating != nil && *ep.Conditions.Terminating { + return false + } return ep.Conditions.Ready == nil || *ep.Conditions.Ready } diff --git a/pkg/l4/state_test.go b/pkg/l4/state_test.go index e82b5a6..8dd18dc 100644 --- a/pkg/l4/state_test.go +++ b/pkg/l4/state_test.go @@ -103,9 +103,11 @@ func TestBuildAnnouncedServiceUsesLocalReadyBackends(t *testing.T) { Ports: []PortKey{key("192.0.2.10", 80), key("192.0.2.10", 443)}, Rules: []Rule{ {PortKey: key("192.0.2.10", 80), Service: "ns/pg", - Backends: []Backend{be("10.0.0.1", 8080), be("10.0.0.2", 8080), be("10.0.0.6", 8080)}}, + Backends: []Backend{be("10.0.0.1", 8080), be("10.0.0.2", 8080), be("10.0.0.6", 8080)}, + Draining: []Backend{be("10.0.0.4", 8080), be("10.0.0.5", 8080)}}, {PortKey: key("192.0.2.10", 443), Service: "ns/pg", - Backends: []Backend{be("10.0.0.1", 8443), be("10.0.0.2", 8443), be("10.0.0.6", 8443)}}, + Backends: []Backend{be("10.0.0.1", 8443), be("10.0.0.2", 8443), be("10.0.0.6", 8443)}, + Draining: []Backend{be("10.0.0.4", 8443), be("10.0.0.5", 8443)}}, }, } if !reflect.DeepEqual(got, want) { @@ -418,3 +420,38 @@ func hasNotice(notices []Notice, service, substr string) bool { } return false } + +// A local endpoint that is still listed but not ready, or terminating, gets no +// new connection. It is kept as draining, so its live connections are not +// purged: a rolling update of an Ingress must not reset every client. +func TestBuildKeepsNotReadyLocalEndpointsAsDraining(t *testing.T) { + svc := l4Service("ns", "web", "192.0.2.10") + svc.Spec.Ports = []v1.ServicePort{svcPort("http", 80)} + + notReady := endpoint("10.0.0.2", thisNode) + notReady.Conditions.Ready = ptr(false) + // publishNotReadyAddresses keeps Ready true on a terminating endpoint. + terminating := endpoint("10.0.0.3", thisNode) + terminating.Conditions.Terminating = ptr(true) + remote := endpoint("10.0.0.4", "node-b") + remote.Conditions.Ready = ptr(false) + + got, _ := Build(Input{ + NodeName: thisNode, + Services: []*v1.Service{svc}, + EndpointSlices: []*discoveryv1.EndpointSlice{slice("ns", "web", "web-1", + []discoveryv1.EndpointPort{epPort("http", 8080)}, + endpoint("10.0.0.1", thisNode), notReady, terminating, remote)}, + Announced: map[string]bool{"ns/web": true}, + }) + + want := []Rule{{ + PortKey: key("192.0.2.10", 80), + Service: "ns/web", + Backends: []Backend{be("10.0.0.1", 8080)}, + Draining: []Backend{be("10.0.0.2", 8080), be("10.0.0.3", 8080)}, + }} + if !reflect.DeepEqual(got.Rules, want) { + t.Errorf("Rules = %+v, want %+v", got.Rules, want) + } +} From dedc066ea2b2dd501b41d172517914f585e3c37d Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:43:16 +0200 Subject: [PATCH 28/42] controllers: keep a forced L4 pass forced until it succeeds The first pass and the periodic resync re-apply the table even when the desired state did not change, which is how a table removed or altered by hand comes back. When such a pass failed, its retry ran as an ordinary one, found the state unchanged and skipped the commit, so the table stayed broken until the next resync, a minute later. Found in review. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/controllers/l4_controller.go | 7 ++++-- pkg/controllers/l4_controller_test.go | 36 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/l4_controller.go b/pkg/controllers/l4_controller.go index 42e8ced..02b7009 100644 --- a/pkg/controllers/l4_controller.go +++ b/pkg/controllers/l4_controller.go @@ -168,16 +168,19 @@ func (c *L4Controller) run(ctx context.Context) { defer resync.Stop() var retry <-chan time.Time - force := true // the first pass replaces whatever the previous instance left + // The first pass replaces whatever the previous instance left. A forced + // pass stays forced until it succeeds: retried as an ordinary one, it would + // find the desired state unchanged and skip the commit it exists for. + force := true for { if err := c.syncWith(c.input(), force); err != nil { l4log.Error(err, "L4 sync failed, retrying") retry = time.After(orDefault(c.RetryInterval, defaultL4RetryInterval)) } else { retry = nil + force = false } - force = false select { case <-ctx.Done(): return diff --git a/pkg/controllers/l4_controller_test.go b/pkg/controllers/l4_controller_test.go index a194831..208fc0d 100644 --- a/pkg/controllers/l4_controller_test.go +++ b/pkg/controllers/l4_controller_test.go @@ -28,6 +28,12 @@ type fakeL4Datapath struct { failing bool } +func (f *fakeL4Datapath) setFailing(v bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.failing = v +} + func (f *fakeL4Datapath) Sync(st l4.State) error { f.mu.Lock() defer f.mu.Unlock() @@ -374,3 +380,33 @@ func TestL4ControllerWithoutNodeNameProgramsNothing(t *testing.T) { t.Errorf("nothing may be programmed without a node name, got %d syncs", dp.count()) } } + +// A forced pass — the first one, or the periodic resync that restores a table +// altered by hand — must stay forced until it succeeds. Retried as an ordinary +// pass, it would find the desired state unchanged and skip the commit. +func TestL4RunKeepsForcingUntilAForcedPassSucceeds(t *testing.T) { + in := l4Input(l4Slice("pg", "10.0.0.1")) + st, _ := l4.Build(in) + dp := &fakeL4Datapath{failing: true} + c := &L4Controller{ + NodeName: l4Node, + Datapath: dp, + Conntrack: &fakePurger{}, + MinSyncInterval: time.Millisecond, + RetryInterval: 5 * time.Millisecond, + ResyncInterval: time.Hour, + applied: &st, // what the datapath last committed, as far as we know + dirty: make(chan struct{}, 1), + input: func() l4.Input { return in }, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { c.run(ctx); close(done) }() + defer func() { cancel(); <-done }() + + time.Sleep(30 * time.Millisecond) + dp.setFailing(false) + waitFor(t, dp, "the forced pass committed on retry", func(got l4.State) bool { + return reflect.DeepEqual(got, st) + }) +} From 1ee6f3d2ba4bc90401145354c4ccfa2ec4be1ebf Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:44:46 +0200 Subject: [PATCH 29/42] proxy: give the L4 sets IDs of their own transaction Rules reference the sets created in the same batch by ID, and github.com/google/nftables draws those IDs from a package-level counter that only each connection's own lock guards. The VM mode queueing its sets on its connection at the same moment could hand two L4 sets the same ID, and a rule would then point at the wrong set. The kernel only needs the IDs unique within a batch, so the table now numbers its own. The purge test now also proves there was a flow to purge, and the conntrack test no longer calls a flow "not a VIP" when the predicate does not look at the VIP. Found in review. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/proxy/conntrack_test.go | 2 +- pkg/proxy/l4_datapath_test.go | 18 ++++++++++++++++-- pkg/proxy/l4_nft.go | 27 +++++++++++++++++++++------ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/pkg/proxy/conntrack_test.go b/pkg/proxy/conntrack_test.go index c9f929a..2a65a73 100644 --- a/pkg/proxy/conntrack_test.go +++ b/pkg/proxy/conntrack_test.go @@ -73,7 +73,7 @@ func TestNetlinkConntrackPurgeDeletesSelectedFlowsOnly(t *testing.T) { createFlow(t, h, "198.51.100.7:40001", "192.0.2.10:80", "10.244.0.10:8080") // backend kept createFlow(t, h, "198.51.100.7:40002", "192.0.2.10:80", "10.244.0.11:8080") // backend gone - createFlow(t, h, "198.51.100.7:40003", "203.0.113.5:443", "10.244.0.11:8443") // not a VIP + createFlow(t, h, "198.51.100.7:40003", "203.0.113.5:443", "10.244.0.11:8443") // same backend IP, other port gone := netip.MustParseAddrPort("10.244.0.11:8080") purger := &NetlinkConntrack{Handle: h} diff --git a/pkg/proxy/l4_datapath_test.go b/pkg/proxy/l4_datapath_test.go index 2d952b9..b6d45de 100644 --- a/pkg/proxy/l4_datapath_test.go +++ b/pkg/proxy/l4_datapath_test.go @@ -290,14 +290,28 @@ func TestL4DatapathPurgesWithdrawnBackend(t *testing.T) { } } + gone := netip.AddrPortFrom(a(backendB), backendPort) + before := 0 + for _, f := range listFlows(t, ct.Handle) { + if f.ReplySrc == gone { + before++ + } + } + if before == 0 { + t.Fatal("no flow towards the backend about to be withdrawn: the test would prove nothing") + } + cur := routerState() cur.Rules[0].Backends = []l4.Backend{bk(backendA, backendPort)} must(t, "Sync", dp.Sync(cur)) - if _, err := ct.Purge(l4.StaleFlows(&prev, cur)); err != nil { + n, err := ct.Purge(l4.StaleFlows(&prev, cur)) + if err != nil { t.Fatalf("Purge: %v", err) } + if int(n) != before { + t.Errorf("Purge deleted %d flows, want the %d towards the withdrawn backend", n, before) + } - gone := netip.AddrPortFrom(a(backendB), backendPort) for _, f := range listFlows(t, ct.Handle) { if f.ReplySrc == gone { t.Errorf("a flow towards the withdrawn backend survived: %+v", f) diff --git a/pkg/proxy/l4_nft.go b/pkg/proxy/l4_nft.go index 6ac5239..8a37315 100644 --- a/pkg/proxy/l4_nft.go +++ b/pkg/proxy/l4_nft.go @@ -114,7 +114,8 @@ func (d *NFTL4Datapath) Teardown() error { // chain masq nat postrouting srcnat-5 // ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { - vips := &nftables.Set{Table: t, Name: "vips", KeyType: nftables.TypeIPAddr} + var ids setIDs + vips := &nftables.Set{Table: t, ID: ids.next(), Name: "vips", KeyType: nftables.TypeIPAddr} if err := conn.AddSet(vips, addrElements(st.VIPs)); err != nil { return fmt.Errorf("could not add set %s: %w", vips.Name, err) } @@ -123,7 +124,7 @@ func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { if err != nil { return fmt.Errorf("could not build vip_ports key type: %w", err) } - vipPorts := &nftables.Set{Table: t, Name: "vip_ports", KeyType: portKeyType, Concatenation: true} + vipPorts := &nftables.Set{Table: t, ID: ids.next(), Name: "vip_ports", KeyType: portKeyType, Concatenation: true} var portElems []nftables.SetElement for _, pk := range st.Ports { portElems = append(portElems, nftables.SetElement{ @@ -134,7 +135,7 @@ func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { return fmt.Errorf("could not add set %s: %w", vipPorts.Name, err) } - nodeIPs := &nftables.Set{Table: t, Name: "node_ips", KeyType: nftables.TypeIPAddr} + nodeIPs := &nftables.Set{Table: t, ID: ids.next(), Name: "node_ips", KeyType: nftables.TypeIPAddr} if err := conn.AddSet(nodeIPs, addrElements(st.NodeIPs)); err != nil { return fmt.Errorf("could not add set %s: %w", nodeIPs.Name, err) } @@ -145,7 +146,7 @@ func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { // but github.com/google/nftables v0.3.0 can only emit it with the generic // conntrack key, which nft then fails to decode: "nft list" aborts on the // whole ruleset, which is too high a price for every operator on the node. - backends := &nftables.Set{Table: t, Name: "backends", KeyType: portKeyType, Concatenation: true} + backends := &nftables.Set{Table: t, ID: ids.next(), Name: "backends", KeyType: portKeyType, Concatenation: true} var backendElems []nftables.SetElement seen := map[l4.Backend]bool{} for _, r := range st.Rules { @@ -201,7 +202,7 @@ func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { // the packet loop through the gateway. exprs = append(exprs, &expr.Verdict{Kind: expr.VerdictDrop}) } else { - m, err := addBackendMap(conn, t, r) + m, err := addBackendMap(conn, t, r, ids.next()) if err != nil { return err } @@ -264,16 +265,30 @@ func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { return nil } +// setIDs hands out the set IDs of one transaction, which the rules use to +// reference sets created in the same batch. github.com/google/nftables would +// draw them from a package-level counter guarded only by each connection's own +// lock, so the VM mode queueing its sets on its connection at the same moment +// could give two of ours the same ID, and a rule would then point at the wrong +// set. The kernel only needs them unique within a batch. +type setIDs uint32 + +func (n *setIDs) next() uint32 { + *n++ + return uint32(*n) +} + // addBackendMap adds the map from round-robin slot to backend for one frontend. // It is named after the frontend, which is unique, and commented with the // service it belongs to. -func addBackendMap(conn *nftables.Conn, t *nftables.Table, r l4.Rule) (*nftables.Set, error) { +func addBackendMap(conn *nftables.Conn, t *nftables.Table, r l4.Rule, id uint32) (*nftables.Set, error) { dataType, err := nftables.ConcatSetType(nftables.TypeIPAddr, nftables.TypeInetService) if err != nil { return nil, fmt.Errorf("could not build backend map data type: %w", err) } m := &nftables.Set{ Table: t, + ID: id, Name: fmt.Sprintf("backends-%s-%s-%d", r.VIP, protoName(r.Protocol), r.Port), Comment: r.Service, IsMap: true, From a83bd5b172870ee74eeaaeb72f93bbea0f60cf64 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:47:51 +0200 Subject: [PATCH 30/42] l4: drain only terminating backends, purge not-ready ones Keeping a not-ready endpoint's flows alive left clients talking to a backend whose own probe says it cannot serve, until the flows timed out. Only a terminating endpoint shuts down on purpose and has connections worth letting finish; a not-ready one is now purged as if it had been removed. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/rfc/l4-loadbalancer-mode.md | 12 +++++++----- pkg/l4/conntrack_test.go | 4 ++-- pkg/l4/state.go | 23 +++++++++++++++-------- pkg/l4/state_test.go | 15 ++++++++------- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index 327b47f..97e3e6d 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -322,14 +322,16 @@ the first sync, the mode deletes the conntrack entries that: destination), and - target a VIP the mode knows (previous or current state), and - point to a (VIP, protocol, port, backend IP, backend port) that is no longer - programmed **on this node** — backend gone from the EndpointSlices, port + programmed **on this node** — backend gone from the EndpointSlices or not + ready, port removed, service removed or no longer managed, or this node no longer the announcer. -A local endpoint that is still listed but not ready, or terminating, is -*draining*: it gets no new connection, but its live ones are kept until it -leaves the slices, so a backend shutting down gracefully (an ingress-nginx -rolling update) can finish them, as behind kube-proxy or Cilium. +A local endpoint that is terminating is *draining*: it gets no new +connection, but its live ones are kept until it leaves the slices, so a backend +shutting down gracefully (an ingress-nginx rolling update) can finish them, as +behind kube-proxy or Cilium. An endpoint that is merely not ready is not: its +probe says it cannot serve, so its flows are purged as if it had been removed. Untranslated flows towards a VIP (a pod on a non-announcer node passing through) are never touched, nor is anything the VM mode tracks. The purge runs after the diff --git a/pkg/l4/conntrack_test.go b/pkg/l4/conntrack_test.go index 9f738a0..9aacf07 100644 --- a/pkg/l4/conntrack_test.go +++ b/pkg/l4/conntrack_test.go @@ -119,8 +119,8 @@ func TestProtocolNumber(t *testing.T) { } } -// A backend that stops being ready keeps its live connections until it leaves -// the EndpointSlice; only then are they purged. +// A terminating backend keeps its live connections until it leaves the +// EndpointSlice; only then are they purged. func TestDrainingBackendKeepsItsFlows(t *testing.T) { ready := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080), be("10.0.0.2", 8080))) draining := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080))) diff --git a/pkg/l4/state.go b/pkg/l4/state.go index 85dfa9e..6453293 100644 --- a/pkg/l4/state.go +++ b/pkg/l4/state.go @@ -35,10 +35,12 @@ type Rule struct { // Service is the namespace/name of the Service the frontend belongs to. Service string Backends []Backend - // Draining are local endpoints still listed but not ready, or - // terminating. They get no new connection, but their live ones are not - // purged: a backend shutting down gracefully must be able to finish them, - // as it would behind kube-proxy or Cilium. The datapath ignores them. + // Draining are the local endpoints that are terminating. They get no new + // connection, but their live ones are not purged: a backend shutting down + // gracefully must be able to finish them, as it would behind kube-proxy or + // Cilium. A merely not-ready endpoint is not draining: its probe says it + // cannot serve, so its flows are purged like a removed one's. The datapath + // ignores this field. Draining []Backend } @@ -163,7 +165,7 @@ func (b *builder) addService(svc *v1.Service) { } // localBackends returns the endpoints of the service port hosted on this node, -// sorted and deduplicated: the ones serving, and the draining ones. The port +// sorted and deduplicated: the ones serving, and the terminating ones. The port // is matched by name and protocol, the way kube-proxy does, which also // resolves named target ports. func (b *builder) localBackends(key, ns, name, portName string, proto v1.Protocol) (ready, draining []Backend) { @@ -187,9 +189,10 @@ func (b *builder) localBackends(key, ns, name, portName string, proto v1.Protoco b.notice(key, "backend %s excluded: it is the pod of VM-mode service %s", ip, owner) continue } - if serves(ep) { + switch { + case serves(ep): ready = append(ready, Backend{IP: ip, Port: port}) - } else { + case terminating(ep): draining = append(draining, Backend{IP: ip, Port: port}) } } @@ -237,12 +240,16 @@ func (b *builder) vmModePods() map[netip.Addr]string { // not serve even when Ready says true, which publishNotReadyAddresses makes it // do. func serves(ep discoveryv1.Endpoint) bool { - if ep.Conditions.Terminating != nil && *ep.Conditions.Terminating { + if terminating(ep) { return false } return ep.Conditions.Ready == nil || *ep.Conditions.Ready } +func terminating(ep discoveryv1.Endpoint) bool { + return ep.Conditions.Terminating != nil && *ep.Conditions.Terminating +} + func slicePort(s *discoveryv1.EndpointSlice, name string, proto v1.Protocol) (uint16, bool) { for _, p := range s.Ports { pName := "" diff --git a/pkg/l4/state_test.go b/pkg/l4/state_test.go index 8dd18dc..5ecf3de 100644 --- a/pkg/l4/state_test.go +++ b/pkg/l4/state_test.go @@ -104,10 +104,10 @@ func TestBuildAnnouncedServiceUsesLocalReadyBackends(t *testing.T) { Rules: []Rule{ {PortKey: key("192.0.2.10", 80), Service: "ns/pg", Backends: []Backend{be("10.0.0.1", 8080), be("10.0.0.2", 8080), be("10.0.0.6", 8080)}, - Draining: []Backend{be("10.0.0.4", 8080), be("10.0.0.5", 8080)}}, + Draining: []Backend{be("10.0.0.5", 8080)}}, {PortKey: key("192.0.2.10", 443), Service: "ns/pg", Backends: []Backend{be("10.0.0.1", 8443), be("10.0.0.2", 8443), be("10.0.0.6", 8443)}, - Draining: []Backend{be("10.0.0.4", 8443), be("10.0.0.5", 8443)}}, + Draining: []Backend{be("10.0.0.5", 8443)}}, }, } if !reflect.DeepEqual(got, want) { @@ -421,10 +421,11 @@ func hasNotice(notices []Notice, service, substr string) bool { return false } -// A local endpoint that is still listed but not ready, or terminating, gets no -// new connection. It is kept as draining, so its live connections are not -// purged: a rolling update of an Ingress must not reset every client. -func TestBuildKeepsNotReadyLocalEndpointsAsDraining(t *testing.T) { +// A terminating local endpoint gets no new connection, but is kept as +// draining, so its live connections are not purged: a rolling update of an +// Ingress must not reset every client. A merely not-ready one is not: its +// probe says it cannot serve, so its flows are purged like a removed one's. +func TestBuildKeepsTerminatingLocalEndpointsAsDraining(t *testing.T) { svc := l4Service("ns", "web", "192.0.2.10") svc.Spec.Ports = []v1.ServicePort{svcPort("http", 80)} @@ -449,7 +450,7 @@ func TestBuildKeepsNotReadyLocalEndpointsAsDraining(t *testing.T) { PortKey: key("192.0.2.10", 80), Service: "ns/web", Backends: []Backend{be("10.0.0.1", 8080)}, - Draining: []Backend{be("10.0.0.2", 8080), be("10.0.0.3", 8080)}, + Draining: []Backend{be("10.0.0.3", 8080)}, }} if !reflect.DeepEqual(got.Rules, want) { t.Errorf("Rules = %+v, want %+v", got.Rules, want) From 181b41c3d330f4011c7044ef1d967b2c7f6eeefa Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 15:48:46 +0200 Subject: [PATCH 31/42] docs: reconcile the L4 mode with the Kyverno guard on CCM Services hikube-gitops !121 proposes ccm-lb-tenant-fields-guard, which refuses on CREATE and UPDATE any CCM Service carrying service.cilium.io/type, so that a tenant cannot drive Cilium through the CCM's copy of its annotations. As proposed it would refuse every L4 CCM Service, and the CCM's own updates of them. The RFC now narrows that one key instead of dropping it: allowed only as ClusterIP, and only on a Service that also carries the lb-proxy label, which the platform sets and the CCM never copies from a tenant. The CCM patch sets both itself, per Service and for eTP Local only, rather than taking the annotation from the tenant. It also states plainly that a tenant cannot switch its own LB to the L4 mode, and why. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/rfc/l4-loadbalancer-mode.md | 91 +++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 6 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index 97e3e6d..eb58114 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -383,9 +383,20 @@ Consequences: distinction must use ClusterIP and network policies, or mTLS. - **ingress-nginx `whitelist-source-range`, Postgres `pg_hba`** and similar application-level allowlists see the same addresses as the network policies. -- **Annotation trust**: see 2.5. A tenant copying `service.cilium.io/type` onto - its infra Service through the CCM only makes its own LB dark: without the - label, cozy-proxy does not pick it up. +- **A tenant cannot switch its own LB to the L4 mode.** The kubevirt CCM + copies every annotation of the tenant Service onto the infra Service, so a + tenant controls `service.cilium.io/type` there — but not the labels: the CCM + sets only its own (`cluster.x-k8s.io/tenant-service-*`, `cluster-name`) and the + platform's `infraLabels`. cozy-proxy requires the + `networking.cozystack.io/lb-proxy` label (2.5), so the annotation alone never + makes it program anything. At worst a tenant copying `service.cilium.io/type: + ClusterIP` makes its own LB dark (Cilium lets go, nobody takes over); the + Kyverno guard of 7.1 refuses such a Service in the first place. +- **What the tenant still controls on an L4 CCM Service**: the ports (the CCM + copies them) and `externalTrafficPolicy`. A tenant Service in eTP `Cluster` + that the platform labelled anyway would be refused by cozy-proxy and stay + dark, which is why the label and the annotation must be set per Service, for + eTP `Local` only (7). ## 6. `externalTrafficPolicy` @@ -408,10 +419,70 @@ enabled per environment. |---|---|---| | Postgres | `packages/apps/postgres/templates/external-svc.yaml` | add the label and the annotation; eTP is already `Local` | | MariaDB | `packages/apps/mariadb/templates/mariadb.yaml` (`spec.service` / `spec.primaryService`) | add the label and the annotation through the operator's service template, and set `externalTrafficPolicy: Local` (operator default is `Cluster`) | -| Tenant Kubernetes (CCM) | `packages/apps/kubernetes/templates/cloud-config.yaml` | label through `infraLabels`, which the CCM already supports. The annotation needs a CCM patch (`infraAnnotations`, next to the existing patches in `packages/apps/kubernetes/images/kubevirt-cloud-provider/patches`). The CCM only sets labels and annotations at creation, so existing infra Services need a one-shot patch. Only tenant Services with eTP `Local` are eligible in phase 1. | +| Tenant Kubernetes (CCM) | CCM patch, next to the existing ones in `packages/apps/kubernetes/images/kubevirt-cloud-provider/patches`, switched on from `packages/apps/kubernetes/templates/cloud-config.yaml` | The CCM itself sets the label and `service.cilium.io/type: ClusterIP`, overriding any value copied from the tenant, and only when the tenant Service is eTP `Local`. `infraLabels` alone is not enough: it labels every Service of the cluster, eTP `Cluster` ones included, and the annotation would still have to come from the tenant. The CCM only sets labels and annotations at creation, so existing infra Services need a one-shot patch. See 7.1 for the Kyverno guard. | | Ingress | `packages/extra/ingress/templates/nginx-ingress.yaml` | `controller.service.labels` / `annotations`; eTP is already `Local`. The host ingress (`tenant-root`) with PROXY protocol and ouroboros is last, see Open questions. | | cozy-proxy | `packages/system/cozy-proxy` | bump the vendored chart, enable the mode, extend RBAC (EndpointSlices, Nodes, ServiceL2Status) | +### 7.1 Coexistence with the Kyverno guard on CCM Services + +The policy `ccm-lb-tenant-fields-guard` (hikube-gitops !121, `Audit` when +proposed) refuses, on CREATE and UPDATE, any Service carrying +`cluster.x-k8s.io/tenant-service-name` that has `spec.externalIPs` or one of a +list of MetalLB, external-dns and Cilium annotations, `service.cilium.io/type` +included: a tenant must not drive them through the CCM's copy. As proposed, it +would refuse every L4 CCM Service — at creation, and on every later UPDATE, the +CCM's own port updates and the one-shot migration patch included. + +The two are reconciled by narrowing that one key rather than dropping it: + +1. `service.cilium.io/type` leaves the guard's generic deny list, and gets a + rule of its own: it is allowed only with the value `ClusterIP` **and** on a + Service that also carries `networking.cozystack.io/lb-proxy: cozy-proxy`. + Every other `service.cilium.io/*` key stays refused. +2. The label is set by the platform only (the CCM patch or the migration + patch); the CCM copies no tenant label, so a tenant cannot satisfy the + exception by itself. +3. The annotation is set by the CCM patch, overriding the tenant's value, rather + than copied from the tenant. A tenant copying `ClusterIP` onto a Service the + platform did not label is still refused. + +A sketch of the dedicated rule (to be validated against the Kyverno version in +use, 1.18.2 when written): + +```yaml +- name: cilium-type-only-for-l4 + match: + any: + - resources: + kinds: ["Service"] + operations: ["CREATE", "UPDATE"] + selector: + matchExpressions: + - {key: cluster.x-k8s.io/tenant-service-name, operator: Exists} + preconditions: + all: + - key: "{{ request.object.metadata.annotations.\"service.cilium.io/type\" || '' }}" + operator: NotEquals + value: "" + validate: + message: >- + service.cilium.io/type is only allowed as ClusterIP on a LoadBalancer the + platform handed to cozy-proxy (label networking.cozystack.io/lb-proxy). + deny: + conditions: + any: + - key: "{{ request.object.metadata.annotations.\"service.cilium.io/type\" }}" + operator: NotEquals + value: ClusterIP + - key: "{{ request.object.metadata.labels.\"networking.cozystack.io/lb-proxy\" || '' }}" + operator: NotEquals + value: cozy-proxy +``` + +Order of the rollout: the narrowed policy goes in before the first CCM Service +is migrated, otherwise the migration patch is refused once the policy is +enforced, and reported as a violation while it is in `Audit`. + ## 8. Migration and rollback, per service Forward, for one service: @@ -432,6 +503,11 @@ ignores it): the runbook must always remove the annotation. Whole mode: remove every annotation, then disable the mode (which deletes the table). +For a CCM Service, the label and the annotation are both put on the infra +Service by the platform (7, 7.1), never through the tenant Service; the rollback +removes the annotation there, and the CCM patch must not put it back on its next +update. + ## 9. Rollout plan Each step runs for at least a week before the next, with the rollback above @@ -484,9 +560,12 @@ another tenant, the source IP logged by the backend, MetalLB announcer, and the 9. **Observability**: metrics (sync duration and errors, programmed services, purged flows) and Events on the Service when it is refused. Phase 1 only logs, once per change. -10. **Where the label lives**: `networking.cozystack.io/lb-proxy: cozy-proxy` is a +10. **Kyverno guard (7.1)**: the narrowed rule is a sketch; it has to be agreed + with the owners of hikube-gitops !121 and tested on the lab, on CREATE and on + the CCM's UPDATE. +11. **Where the label lives**: `networking.cozystack.io/lb-proxy: cozy-proxy` is a proposal; the maintainers may prefer another key. -11. **`google/nftables` limits**: no `typeof` on sets, and no direction on the +12. **`google/nftables` limits**: no `typeof` on sets, and no direction on the typed conntrack keys (3.1). Both are cosmetic for `nft list` today; fixing them upstream would let the masquerade match the original destination. From b003f67cb528ddab77a951a1fdb39bc4df3cdfd4 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 16:31:08 +0200 Subject: [PATCH 32/42] proxy: rename the L4 dnat chain so nft can name it Found on the lab: "nft list chain ip cozy_proxy_l4 dnat" is a syntax error, since nft reads dnat as its keyword. Operators inspect one chain at a time, so the chain is now called "translate". The test lists every chain of the table by name with the nft binary. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/rfc/l4-loadbalancer-mode.md | 6 +++--- pkg/proxy/l4_nft.go | 10 ++++++---- pkg/proxy/l4_nft_test.go | 27 +++++++++++++++++++++++++++ pkg/proxy/testdata/l4/announcer.nft | 2 +- pkg/proxy/testdata/l4/empty.nft | 2 +- pkg/proxy/testdata/l4/guard-only.nft | 2 +- 6 files changed, 39 insertions(+), 10 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index eb58114..2065c0d 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -200,7 +200,7 @@ table ip cozy_proxy_l4 { ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop } - chain dnat { + chain translate { type nat hook prerouting priority dstnat - 5; policy accept; # announcer only, one rule per (VIP, port) ip daddr tcp dport dnat ip addr . port to numgen inc mod map @backends-//tcp/ @@ -221,7 +221,7 @@ Notes: ICMP errors about a translated flow (PMTU) still reach the NAT and are translated with it. New packets to a VIP on an undeclared (VIP, protocol, port) are dropped, which includes ICMP echo: nobody would answer it anyway. -- `dnat` only holds rules on the announcer. Elsewhere the packet is left alone +- `translate` only holds rules on the announcer. Elsewhere the packet is left alone and routed out, to the announcer. - `masq` only matches flows this node translated (`ct status dnat`) towards one of its L4 targets, from a node IP, so a node process talking to a backend pod @@ -246,7 +246,7 @@ Notes: | -200 | prerouting | | conntrack | flow lookup / creation | | -150 (mangle) | prerouting | `cozy_proxy` | `ingress_dnat` | VM mode: stateless destination rewrite VM VIP -> pod IP | | -140 | prerouting | `cozy_proxy_l4` | `guard` | L4: drop undeclared traffic to L4 VIPs | -| -105 | prerouting | `cozy_proxy_l4` | `dnat` | L4: stateful DNAT, first packet only | +| -105 | prerouting | `cozy_proxy_l4` | `translate` | L4: stateful DNAT, first packet only | | -100 | prerouting | | kube-ovn / iptables nat | unchanged | | 0 (filter) | prerouting | `cozy_proxy` | `port_filter` | VM mode: per-port filter on VM pod IPs | | 95 | postrouting | `cozy_proxy_l4` | `masq` | L4: masquerade node sources | diff --git a/pkg/proxy/l4_nft.go b/pkg/proxy/l4_nft.go index 8a37315..a9a6537 100644 --- a/pkg/proxy/l4_nft.go +++ b/pkg/proxy/l4_nft.go @@ -108,7 +108,7 @@ func (d *NFTL4Datapath) Teardown() error { // chain guard filter prerouting mangle+10 // ct state established,related accept // ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop -// chain dnat nat prerouting dstnat-5 (rules on the announcer only) +// chain translate nat prerouting dstnat-5 (rules on the announcer only) // ip daddr V tcp dport P dnat ip addr . port to numgen inc mod N map @backends-... // ip daddr V tcp dport P drop (announced port with no local backend) // chain masq nat postrouting srcnat-5 @@ -188,8 +188,10 @@ func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { &expr.Verdict{Kind: expr.VerdictDrop}, }}) - dnat := conn.AddChain(&nftables.Chain{ - Name: "dnat", + // Not "dnat": nft takes that for its keyword, and an operator could not + // name the chain on the command line without quoting it. + translate := conn.AddChain(&nftables.Chain{ + Name: "translate", Table: t, Type: nftables.ChainTypeNAT, Hooknum: nftables.ChainHookPrerouting, @@ -222,7 +224,7 @@ func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { } conn.AddRule(&nftables.Rule{ Table: t, - Chain: dnat, + Chain: translate, Exprs: exprs, UserData: userdata.AppendString(nil, userdata.TypeComment, r.Service), }) diff --git a/pkg/proxy/l4_nft_test.go b/pkg/proxy/l4_nft_test.go index 72ec36d..89c5cc5 100644 --- a/pkg/proxy/l4_nft_test.go +++ b/pkg/proxy/l4_nft_test.go @@ -231,3 +231,30 @@ func TestL4TeardownLeavesOtherTablesAlone(t *testing.T) { t.Errorf("tables after Teardown = %v, want only cozy_proxy", names) } } + +// Operators inspect one chain at a time. A chain named after an nft keyword +// ("dnat") cannot be named on the command line without quoting, which is how +// this was found on the lab. +func TestL4ChainsCanBeListedByName(t *testing.T) { + ns := scratchNetns(t) + if err := datapathIn(ns).Sync(announcerState); err != nil { + t.Fatalf("Sync: %v", err) + } + conn, err := nftables.New(nftables.WithNetNSFd(int(ns))) + if err != nil { + t.Fatalf("nftables.New: %v", err) + } + chains, err := conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4) + if err != nil { + t.Fatalf("ListChains: %v", err) + } + nft := nftBinary(t) + for _, ch := range chains { + if ch.Table.Name != L4TableName { + continue + } + if out, err := inNetns(t, ns, nft, "list", "chain", "ip", L4TableName, ch.Name); err != nil { + t.Errorf("nft list chain %s: %v\n%s", ch.Name, err, out) + } + } +} diff --git a/pkg/proxy/testdata/l4/announcer.nft b/pkg/proxy/testdata/l4/announcer.nft index ef5fb32..e7780cb 100644 --- a/pkg/proxy/testdata/l4/announcer.nft +++ b/pkg/proxy/testdata/l4/announcer.nft @@ -46,7 +46,7 @@ table ip cozy_proxy_l4 { ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop } - chain dnat { + chain translate { type nat hook prerouting priority dstnat - 5; policy accept; ip daddr 192.0.2.10 tcp dport 80 dnat ip to numgen inc mod 2 map @backends-192.0.2.10-tcp-80 comment "tenant-a/ingress" ip daddr 192.0.2.10 tcp dport 443 dnat ip to numgen inc mod 2 map @backends-192.0.2.10-tcp-443 comment "tenant-a/ingress" diff --git a/pkg/proxy/testdata/l4/empty.nft b/pkg/proxy/testdata/l4/empty.nft index 9408bd7..f6524b9 100644 --- a/pkg/proxy/testdata/l4/empty.nft +++ b/pkg/proxy/testdata/l4/empty.nft @@ -21,7 +21,7 @@ table ip cozy_proxy_l4 { ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop } - chain dnat { + chain translate { type nat hook prerouting priority dstnat - 5; policy accept; } diff --git a/pkg/proxy/testdata/l4/guard-only.nft b/pkg/proxy/testdata/l4/guard-only.nft index 0ac1af7..02f2e02 100644 --- a/pkg/proxy/testdata/l4/guard-only.nft +++ b/pkg/proxy/testdata/l4/guard-only.nft @@ -28,7 +28,7 @@ table ip cozy_proxy_l4 { ip daddr @vips ip daddr . meta l4proto . th dport != @vip_ports drop } - chain dnat { + chain translate { type nat hook prerouting priority dstnat - 5; policy accept; } From dc0eef220bce064e8ed04748e3f7042889af917f Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 16:31:38 +0200 Subject: [PATCH 33/42] controllers: log the L4 mode under its own name The L4 controller's logger was derived from the VM controller's, so on the lab every L4 line read "services-controller.l4", as if the VM mode were at work. It is now "l4-controller". Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/controllers/l4_controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/l4_controller.go b/pkg/controllers/l4_controller.go index 02b7009..e56b15f 100644 --- a/pkg/controllers/l4_controller.go +++ b/pkg/controllers/l4_controller.go @@ -16,12 +16,16 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + ctrl "sigs.k8s.io/controller-runtime" "github.com/cozystack/cozy-proxy/pkg/l4" nat "github.com/cozystack/cozy-proxy/pkg/proxy" ) -var l4log = log.WithName("l4") +// l4log is not derived from the VM controller's logger: named +// "services-controller.l4", its lines read as if the VM mode were doing the +// work, which is confusing when both modes run in one process. +var l4log = ctrl.Log.WithName("l4-controller") // serviceL2StatusGVR is MetalLB's record of which node announces a service in // L2 mode. Each speaker writes one per service it announces, labelled with its From 3d6921614aa5beb2970e0b985f2fd021c1fa6de7 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 16:42:31 +0200 Subject: [PATCH 34/42] docs: record the lab run of the L4 mode The RFC now lists what the first lab run of the prototype showed, case by case, with the source address each backend saw, and adds the open question it raised: purging the TCP flows of a backend that is still alive leaves an idle client hanging until its own timeout, which kube-proxy avoids by only purging UDP. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/rfc/l4-loadbalancer-mode.md | 35 +++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index 2065c0d..459d070 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -565,7 +565,15 @@ another tenant, the source IP logged by the backend, MetalLB announcer, and the the CCM's UPDATE. 11. **Where the label lives**: `networking.cozystack.io/lb-proxy: cozy-proxy` is a proposal; the maintainers may prefer another key. -12. **`google/nftables` limits**: no `typeof` on sets, and no direction on the +12. **Purging the TCP flows of a backend that is still alive.** On the lab, a + backend taken out of the Service (label removed) while a request was in + flight lost its flow at the purge, and the client, idle while waiting for + the answer, hung until its own timeout: the backend's reply no longer + matched any conntrack entry, and no RST reached the client. kube-proxy + does not purge TCP flows on endpoint removal; it only does so for UDP. + Should the L4 mode purge only UDP, and TCP only when the service, the + port or the announcement goes? +13. **`google/nftables` limits**: no `typeof` on sets, and no direction on the typed conntrack keys (3.1). Both are cosmetic for `nft list` today; fixing them upstream would let the masquerade match the original destination. @@ -592,5 +600,26 @@ docker run --rm --privileged -v "$PWD":/src -w /src golang:1.26 sh -c \ ``` Not in phase 1: UDP, eTP `Cluster`, IPv6, terminating endpoints, metrics and -Events, and a lab validation of the binary itself (the model was validated with -hand-written rules). +Events. + +Lab run on 2026-10-02 (image built from `181b41c`, as a second release with the +VM mode off, next to the platform's cozy-proxy): + +| Case | Result | Source seen by the backend | +|---|---|---| +| Internet, both ports (port translation 80 -> 8080, 81 -> 9090) | OK | the client's public IP | +| pods on the 3 nodes, another tenant | OK | `100.64.0.x` of the announcer | +| VM-mode client on another node, and on the announcer | OK | the VM's public IP | +| round-robin between the announcer's two local backends | OK | | +| undeclared port, ping (tcpdump on the announcer and on another node) | dropped, no packet sent back out | | +| CIDR deny `0.0.0.0/0 except ` on the backend | pods and VM refused, the allowed client passes | | +| backend withdrawn | its flows purged, new connections go to the others | | +| announcer moved by MetalLB | one failed probe out of ~110 (0.5 s interval) | | +| rollout restart of the DaemonSet | open connection kept, 0 failed probes out of 113 | | +| Postgres app, psql from pods, VM and VPN; `pg_stat_activity` | OK | `100.64.0.x`, VM IP, VPN address | +| CNPG switchover | ~7 s unavailable, mostly the server shutting down | | +| rollback (annotation removed) | Cilium serves the VIP again, other tenants refused again | | + +Two bugs were found and fixed: a chain named after an nft keyword (`dnat`), +which nft could not name on the command line, and a misleading logger name. + From c1a064200562a34dc79374b9b0916a362982da12 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 16:47:21 +0200 Subject: [PATCH 35/42] l4: purge TCP flows with their frontend only, as kube-proxy does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the lab, a backend taken out of a Service while a request was in flight lost its flow at the purge, and the client, idle while waiting for the answer, hung until its own timeout: the backend's reply matched no conntrack entry any more and no RST reached the client. Left alone, the request would have completed. kube-proxy only purges UDP flows on endpoint removal. The L4 mode now does the same: a TCP flow is purged when its frontend goes — the service, the port, or this node's announcement, a MetalLB failover included — and a withdrawn TCP backend keeps its established connections until they end, or until its pod goes. UDP keeps the per-backend purge, without which an active flow would follow a backend that left forever. The RFC records the decision and closes the open question. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/rfc/l4-loadbalancer-mode.md | 62 ++++++++++++++------------- pkg/controllers/l4_controller_test.go | 35 ++++++++++++--- pkg/l4/conntrack.go | 57 +++++++++++++++++++----- pkg/l4/conntrack_test.go | 51 ++++++++++++++++------ pkg/proxy/l4_datapath_test.go | 52 +++++++++++++--------- 5 files changed, 176 insertions(+), 81 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index 459d070..63892f6 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -311,27 +311,32 @@ by the previous instance stays in place and keeps forwarding. ### 4.3 Conntrack purge -A conntrack entry outlives the rule that created it. When a backend disappears, -its existing flows would keep being translated to a dead pod until they time out -(hours for TCP), and UDP flows would never recover. - -After every successful sync whose programmed backends changed, and once after -the first sync, the mode deletes the conntrack entries that: - -- were translated by this node (reply source differs from the original - destination), and -- target a VIP the mode knows (previous or current state), and -- point to a (VIP, protocol, port, backend IP, backend port) that is no longer - programmed **on this node** — backend gone from the EndpointSlices or not - ready, port - removed, service removed or no longer managed, or this node no longer the - announcer. - -A local endpoint that is terminating is *draining*: it gets no new -connection, but its live ones are kept until it leaves the slices, so a backend -shutting down gracefully (an ingress-nginx rolling update) can finish them, as -behind kube-proxy or Cilium. An endpoint that is merely not ready is not: its -probe says it cannot serve, so its flows are purged as if it had been removed. +A conntrack entry outlives the rule that created it. The purge follows +kube-proxy. After every successful sync that withdrew something, and once after +the first sync, the mode deletes the conntrack entries that were translated by +this node (reply source differs from the original destination), towards a VIP +the mode knows (previous or current state), and: + +- **TCP**: whose frontend is gone **on this node** — service removed or no + longer managed, port removed, or this node no longer the announcer (MetalLB + moved the VIP). A TCP backend withdrawn from a frontend that remains keeps its + established connections until they end on their own, or until its pod goes; + new connections no longer reach it. +- **UDP** (and any other protocol): whose backend is no longer programmed nor + draining — gone from the EndpointSlices or not ready — or whose frontend is + gone. Without the purge an active UDP flow would keep being sent to a backend + that left, forever. + +Why TCP is left alone: kube-proxy only purges UDP on endpoint removal, and the +lab showed the cost of doing otherwise. A backend taken out of a Service while +a request was in flight lost its flow at the purge, and the client, idle while +waiting for the answer, hung until its own timeout: the backend's reply matched +no entry any more, and no RST reached the client. Left alone, the request +completes. + +A local UDP endpoint that is terminating is *draining*: it gets no new flow, but +its live ones are kept until it leaves the slices. An endpoint that is merely +not ready is not: its probe says it cannot serve. Untranslated flows towards a VIP (a pod on a non-announcer node passing through) are never touched, nor is anything the VM mode tracks. The purge runs after the @@ -565,14 +570,11 @@ another tenant, the source IP logged by the backend, MetalLB announcer, and the the CCM's UPDATE. 11. **Where the label lives**: `networking.cozystack.io/lb-proxy: cozy-proxy` is a proposal; the maintainers may prefer another key. -12. **Purging the TCP flows of a backend that is still alive.** On the lab, a - backend taken out of the Service (label removed) while a request was in - flight lost its flow at the purge, and the client, idle while waiting for - the answer, hung until its own timeout: the backend's reply no longer - matched any conntrack entry, and no RST reached the client. kube-proxy - does not purge TCP flows on endpoint removal; it only does so for UDP. - Should the L4 mode purge only UDP, and TCP only when the service, the - port or the announcement goes? +12. **Purging the TCP flows of a backend that is still alive — decided.** Aligned + on kube-proxy (4.3): on endpoint removal only UDP flows are purged; TCP flows + go with their frontend (service, port, announcement, including a MetalLB + failover). Reason: kube-proxy behaves so, and on the lab an idle client of a + withdrawn but live backend hung without a RST until its own timeout. 13. **`google/nftables` limits**: no `typeof` on sets, and no direction on the typed conntrack keys (3.1). Both are cosmetic for `nft list` today; fixing them upstream would let the masquerade match the original destination. @@ -613,7 +615,7 @@ VM mode off, next to the platform's cozy-proxy): | round-robin between the announcer's two local backends | OK | | | undeclared port, ping (tcpdump on the announcer and on another node) | dropped, no packet sent back out | | | CIDR deny `0.0.0.0/0 except ` on the backend | pods and VM refused, the allowed client passes | | -| backend withdrawn | its flows purged, new connections go to the others | | +| backend withdrawn | its flows purged, new connections go to the others (built from `181b41c`; since then TCP flows of a withdrawn backend are kept, see 4.3) | | | announcer moved by MetalLB | one failed probe out of ~110 (0.5 s interval) | | | rollout restart of the DaemonSet | open connection kept, 0 failed probes out of 113 | | | Postgres app, psql from pods, VM and VPN; `pg_stat_activity` | OK | `100.64.0.x`, VM IP, VPN address | diff --git a/pkg/controllers/l4_controller_test.go b/pkg/controllers/l4_controller_test.go index 208fc0d..cc72c3b 100644 --- a/pkg/controllers/l4_controller_test.go +++ b/pkg/controllers/l4_controller_test.go @@ -151,7 +151,14 @@ func flowTo(backend string) l4.Flow { // The first sync programs the state Build computes and purges once, since the // previous instance may have left translations this one no longer programs. func TestL4SyncProgramsAndPurgesOnFirstSync(t *testing.T) { - dp, ct := &fakeL4Datapath{}, &fakePurger{flows: []l4.Flow{flowTo("10.0.0.1:5432"), flowTo("10.0.0.9:5432")}} + // A flow the previous instance translated through a port the service no + // longer has. + oldPort := l4.Flow{ + Protocol: 6, + OrigDst: netip.MustParseAddrPort("192.0.2.10:5433"), + ReplySrc: netip.MustParseAddrPort("10.0.0.1:5433"), + } + dp, ct := &fakeL4Datapath{}, &fakePurger{flows: []l4.Flow{flowTo("10.0.0.1:5432"), flowTo("10.0.0.9:5432"), oldPort}} c := &L4Controller{NodeName: l4Node, Datapath: dp, Conntrack: ct} in := l4Input(l4Slice("pg", "10.0.0.1")) @@ -166,8 +173,10 @@ func TestL4SyncProgramsAndPurgesOnFirstSync(t *testing.T) { if ct.calls != 1 { t.Fatalf("the first sync must purge once, got %d", ct.calls) } - if len(ct.purged) != 1 || ct.purged[0] != flowTo("10.0.0.9:5432") { - t.Errorf("purged %+v, want only the flow to the backend no longer programmed", ct.purged) + // A TCP flow to a frontend still served is kept whatever its backend, as + // kube-proxy would. + if len(ct.purged) != 1 || ct.purged[0] != oldPort { + t.Errorf("purged %+v, want only the flow through the port that is gone", ct.purged) } } @@ -196,7 +205,9 @@ func TestL4SyncSkipsUnchangedState(t *testing.T) { } } -func TestL4SyncPurgesWhenABackendIsWithdrawn(t *testing.T) { +// A TCP backend withdrawn from a frontend that remains keeps its connections +// until they end; its flows only go with the frontend. +func TestL4SyncPurgesTCPFlowsWithTheirFrontendOnly(t *testing.T) { dp, ct := &fakeL4Datapath{}, &fakePurger{} c := &L4Controller{NodeName: l4Node, Datapath: dp, Conntrack: ct} @@ -213,12 +224,22 @@ func TestL4SyncPurgesWhenABackendIsWithdrawn(t *testing.T) { t.Errorf("adding a backend must not purge, got %d purges", ct.calls) } - // A backend withdrawn: its flows go. + // A backend withdrawn: its connections are left alone. if err := c.syncWith(l4Input(l4Slice("pg", "10.0.0.1", "10.0.0.3")), false); err != nil { t.Fatalf("sync: %v", err) } - if ct.calls != 2 || len(ct.purged) != 1 || ct.purged[0] != flowTo("10.0.0.2:5432") { - t.Errorf("withdrawing a backend must purge its flows, got %d purges, purged %+v", ct.calls, ct.purged) + if ct.calls != 1 || len(ct.purged) != 0 { + t.Errorf("withdrawing a TCP backend must not purge, got %d purges, purged %+v", ct.calls, ct.purged) + } + + // The node stops announcing the service: every flow through it goes. + gone := l4Input(l4Slice("pg", "10.0.0.1", "10.0.0.3")) + gone.Announced = nil + if err := c.syncWith(gone, false); err != nil { + t.Fatalf("sync: %v", err) + } + if ct.calls != 2 || len(ct.purged) != 2 { + t.Errorf("losing the announcement must purge every flow, got %d purges, purged %+v", ct.calls, ct.purged) } } diff --git a/pkg/l4/conntrack.go b/pkg/l4/conntrack.go index 8835770..38a2542 100644 --- a/pkg/l4/conntrack.go +++ b/pkg/l4/conntrack.go @@ -40,18 +40,41 @@ func targets(st State) map[target]struct{} { return out } +// frontend is a (VIP, protocol, port) this node translates. +type frontend struct { + vip netip.Addr + protocol uint8 + port uint16 +} + +func frontends(st State) map[frontend]struct{} { + out := map[frontend]struct{}{} + for _, r := range st.Rules { + out[frontend{r.VIP, protocolNumber(r.Protocol), r.Port}] = struct{}{} + } + return out +} + // StaleFlows returns the predicate selecting the conntrack entries to delete // once cur has been committed, coming from prev (nil after a restart). // -// A conntrack entry outlives the rule that created it: without the purge, the -// flows of a backend that went away keep being translated to it until they -// time out — hours for TCP, never for an active UDP flow. +// A conntrack entry outlives the rule that created it. It is stale when this +// node translated it (its replies do not come from the address the client +// used), towards a VIP the L4 mode knows, and: +// +// - for TCP, when its frontend — the service, the port, or this node's +// announcement — is gone. A TCP connection to a backend withdrawn from a +// frontend that still exists is left to end on its own, or with its pod, +// as kube-proxy does. Purging it helps nobody: on the lab an idle client +// then hung until its own timeout, the backend's reply matching no entry +// and no RST reaching the client. +// - for UDP and the rest, when its backend is no longer programmed nor +// draining: without the purge an active UDP flow would keep being sent to +// a backend that left, forever. // -// An entry is stale when this node translated it (its replies do not come from -// the address the client used), towards a VIP the L4 mode knows, and to a -// target the node no longer programs nor drains. Untranslated flows to a VIP — a pod on a -// node that does not announce it, passing through — are left alone, and so is -// everything addressed elsewhere, which keeps the VM mode out of reach. +// Untranslated flows to a VIP — a pod on a node that does not announce it, +// passing through — are left alone, and so is everything addressed elsewhere, +// which keeps the VM mode out of reach. func StaleFlows(prev *State, cur State) func(Flow) bool { vips := map[netip.Addr]struct{}{} for _, st := range []*State{prev, &cur} { @@ -66,6 +89,7 @@ func StaleFlows(prev *State, cur State) func(Flow) bool { } } keep := targets(cur) + served := frontends(cur) return func(f Flow) bool { if _, ours := vips[f.OrigDst.Addr()]; !ours { @@ -74,21 +98,34 @@ func StaleFlows(prev *State, cur State) func(Flow) bool { if f.ReplySrc == f.OrigDst { return false } + if f.Protocol == protocolNumber(v1.ProtocolTCP) { + _, stillServed := served[frontend{f.OrigDst.Addr(), f.Protocol, f.OrigDst.Port()}] + return !stillServed + } _, programmed := keep[target{f.OrigDst.Addr(), f.Protocol, f.OrigDst.Port(), f.ReplySrc}] return !programmed } } -// PurgeNeeded reports whether moving from prev to cur withdrew a translation, -// which is when conntrack may hold entries StaleFlows would select. After a +// PurgeNeeded reports whether moving from prev to cur withdrew something whose +// flows StaleFlows would select: a frontend, or a non-TCP backend. After a // restart (prev nil) the answer is always yes: changes may have happened while // no instance was watching. func PurgeNeeded(prev *State, cur State) bool { if prev == nil { return true } + served := frontends(cur) + for fe := range frontends(*prev) { + if _, ok := served[fe]; !ok { + return true + } + } now := targets(cur) for t := range targets(*prev) { + if t.protocol == protocolNumber(v1.ProtocolTCP) { + continue + } if _, ok := now[t]; !ok { return true } diff --git a/pkg/l4/conntrack_test.go b/pkg/l4/conntrack_test.go index 9aacf07..dd47437 100644 --- a/pkg/l4/conntrack_test.go +++ b/pkg/l4/conntrack_test.go @@ -34,15 +34,28 @@ func rule(vip string, port uint16, backends ...Backend) Rule { return Rule{PortKey: key(vip, port), Service: "ns/svc", Backends: backends} } +func udpRule(vip string, port uint16, backends ...Backend) Rule { + r := rule(vip, port, backends...) + r.Protocol = v1.ProtocolUDP + return r +} + +// The purge follows kube-proxy: a withdrawn endpoint loses its UDP flows, but +// its TCP connections are left to end on their own; TCP flows only go with +// their frontend — the service, the port, or this node's announcement. func TestStaleFlows(t *testing.T) { prev := stateWith( rule("192.0.2.10", 80, be("10.0.0.1", 8080), be("10.0.0.2", 8080)), rule("192.0.2.10", 443, be("10.0.0.1", 8443)), + udpRule("192.0.2.10", 53, be("10.0.0.1", 5353), be("10.0.0.2", 5353)), rule("192.0.2.20", 5432, be("10.0.0.3", 5432)), ) // 10.0.0.2 went away, port 443 was removed from the service, and the // 192.0.2.20 service is gone altogether. - cur := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080))) + cur := stateWith( + rule("192.0.2.10", 80, be("10.0.0.1", 8080)), + udpRule("192.0.2.10", 53, be("10.0.0.1", 5353)), + ) stale := StaleFlows(&prev, cur) @@ -52,10 +65,13 @@ func TestStaleFlows(t *testing.T) { want bool }{ {"backend still programmed", translated(tcp, "192.0.2.10:80", "10.0.0.1:8080"), false}, - {"backend gone", translated(tcp, "192.0.2.10:80", "10.0.0.2:8080"), true}, + // Purging it would leave an idle client hanging without a RST, as + // seen on the lab; the connection ends on its own, or with its pod. + {"TCP backend gone, frontend kept", translated(tcp, "192.0.2.10:80", "10.0.0.2:8080"), false}, + {"UDP backend gone", translated(udp, "192.0.2.10:53", "10.0.0.2:5353"), true}, + {"UDP backend still programmed", translated(udp, "192.0.2.10:53", "10.0.0.1:5353"), false}, {"port removed", translated(tcp, "192.0.2.10:443", "10.0.0.1:8443"), true}, {"service removed", translated(tcp, "192.0.2.20:5432", "10.0.0.3:5432"), true}, - {"same backend IP, other port", translated(tcp, "192.0.2.10:80", "10.0.0.1:9090"), true}, {"protocol not programmed", translated(udp, "192.0.2.10:80", "10.0.0.1:8080"), true}, // A pod on a node that does not announce the VIP goes through // untranslated; its flow is not ours to cut. @@ -72,8 +88,8 @@ func TestStaleFlows(t *testing.T) { } // After a restart there is no previous state: whatever this node translated -// towards a VIP it knows and no longer programs must go — for instance after -// the announcement moved to another node while the pod was down. +// towards a frontend it no longer programs must go — for instance after the +// announcement moved to another node while the pod was down. func TestStaleFlowsWithoutPreviousState(t *testing.T) { cur := State{ VIPs: []netip.Addr{addr("192.0.2.10")}, @@ -83,6 +99,10 @@ func TestStaleFlowsWithoutPreviousState(t *testing.T) { if !stale(translated(tcp, "192.0.2.10:80", "10.0.0.1:8080")) { t.Error("a translation this node no longer programs must be purged") } + announced := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080))) + if StaleFlows(nil, announced)(translated(tcp, "192.0.2.10:80", "10.0.0.9:8080")) { + t.Error("a TCP flow to a frontend still programmed must be kept, whatever its backend") + } if stale(translated(tcp, "192.0.2.10:80", "192.0.2.10:80")) { t.Error("an untranslated flow must be kept") } @@ -100,7 +120,10 @@ func TestPurgeNeeded(t *testing.T) { {"unchanged", &base, base, false}, {"backend added", &base, stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080), be("10.0.0.2", 8080))), false}, {"service added", &base, stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080)), rule("192.0.2.11", 80)), false}, - {"backend replaced", &base, stateWith(rule("192.0.2.10", 80, be("10.0.0.2", 8080))), true}, + {"TCP backend replaced", &base, stateWith(rule("192.0.2.10", 80, be("10.0.0.2", 8080))), false}, + {"UDP backend replaced", ptr(stateWith(udpRule("192.0.2.10", 53, be("10.0.0.1", 5353)))), + stateWith(udpRule("192.0.2.10", 53, be("10.0.0.2", 5353))), true}, + {"port removed", &base, stateWith(rule("192.0.2.10", 81, be("10.0.0.1", 8080))), true}, {"no longer announced", &base, State{VIPs: base.VIPs, Ports: base.Ports}, true}, {"node IPs only", &base, State{VIPs: base.VIPs, Ports: base.Ports, Rules: base.Rules, NodeIPs: []netip.Addr{addr("10.200.24.11")}}, false}, } @@ -119,24 +142,24 @@ func TestProtocolNumber(t *testing.T) { } } -// A terminating backend keeps its live connections until it leaves the -// EndpointSlice; only then are they purged. +// A terminating UDP backend keeps its flows until it leaves the EndpointSlice; +// only then are they purged. func TestDrainingBackendKeepsItsFlows(t *testing.T) { - ready := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080), be("10.0.0.2", 8080))) - draining := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080))) - draining.Rules[0].Draining = []Backend{be("10.0.0.2", 8080)} - gone := stateWith(rule("192.0.2.10", 80, be("10.0.0.1", 8080))) + ready := stateWith(udpRule("192.0.2.10", 53, be("10.0.0.1", 5353), be("10.0.0.2", 5353))) + draining := stateWith(udpRule("192.0.2.10", 53, be("10.0.0.1", 5353))) + draining.Rules[0].Draining = []Backend{be("10.0.0.2", 5353)} + gone := stateWith(udpRule("192.0.2.10", 53, be("10.0.0.1", 5353))) if PurgeNeeded(&ready, draining) { t.Error("a backend that starts draining must not trigger a purge") } - if StaleFlows(&ready, draining)(translated(tcp, "192.0.2.10:80", "10.0.0.2:8080")) { + if StaleFlows(&ready, draining)(translated(udp, "192.0.2.10:53", "10.0.0.2:5353")) { t.Error("a draining backend's flows must be kept") } if !PurgeNeeded(&draining, gone) { t.Error("a draining backend that leaves the slice must trigger a purge") } - if !StaleFlows(&draining, gone)(translated(tcp, "192.0.2.10:80", "10.0.0.2:8080")) { + if !StaleFlows(&draining, gone)(translated(udp, "192.0.2.10:53", "10.0.0.2:5353")) { t.Error("the flows of a backend gone from the slice must be purged") } } diff --git a/pkg/proxy/l4_datapath_test.go b/pkg/proxy/l4_datapath_test.go index b6d45de..cb014c4 100644 --- a/pkg/proxy/l4_datapath_test.go +++ b/pkg/proxy/l4_datapath_test.go @@ -274,9 +274,11 @@ func TestL4DatapathDropsPortWithoutBackend(t *testing.T) { } } -// After a backend is withdrawn, its existing flows are purged from the -// announcer's conntrack, and new connections only reach the remaining one. -func TestL4DatapathPurgesWithdrawnBackend(t *testing.T) { +// A TCP backend withdrawn from a frontend that remains keeps its flows, as +// behind kube-proxy, while new connections only reach the remaining backend. +// The flows go once the frontend goes — here, the node no longer announcing +// the VIP. +func TestL4DatapathPurgesTCPFlowsWithTheirFrontend(t *testing.T) { tp := newTopology(t) serve(t, tp.server) dp := datapathIn(tp.router) @@ -290,32 +292,28 @@ func TestL4DatapathPurgesWithdrawnBackend(t *testing.T) { } } - gone := netip.AddrPortFrom(a(backendB), backendPort) - before := 0 - for _, f := range listFlows(t, ct.Handle) { - if f.ReplySrc == gone { - before++ + withdrawn := netip.AddrPortFrom(a(backendB), backendPort) + flowsTo := func(b netip.AddrPort) int { + n := 0 + for _, f := range listFlows(t, ct.Handle) { + if f.ReplySrc == b { + n++ + } } + return n } - if before == 0 { + if flowsTo(withdrawn) == 0 { t.Fatal("no flow towards the backend about to be withdrawn: the test would prove nothing") } cur := routerState() cur.Rules[0].Backends = []l4.Backend{bk(backendA, backendPort)} must(t, "Sync", dp.Sync(cur)) - n, err := ct.Purge(l4.StaleFlows(&prev, cur)) - if err != nil { - t.Fatalf("Purge: %v", err) + if n, err := ct.Purge(l4.StaleFlows(&prev, cur)); err != nil || n != 0 { + t.Fatalf("Purge after a TCP backend withdrawal = %d, %v; want nothing purged", n, err) } - if int(n) != before { - t.Errorf("Purge deleted %d flows, want the %d towards the withdrawn backend", n, before) - } - - for _, f := range listFlows(t, ct.Handle) { - if f.ReplySrc == gone { - t.Errorf("a flow towards the withdrawn backend survived: %+v", f) - } + if flowsTo(withdrawn) == 0 { + t.Error("the withdrawn TCP backend lost its flows; they must end on their own") } for i := 0; i < 2; i++ { ans, err := dial(t, tp.client, testVIP+":80") @@ -326,6 +324,20 @@ func TestL4DatapathPurgesWithdrawnBackend(t *testing.T) { t.Errorf("connection reached %s, want only %s", ans.backend, backendA) } } + + unannounced := routerState() + unannounced.Rules = nil + must(t, "Sync", dp.Sync(unannounced)) + n, err := ct.Purge(l4.StaleFlows(&cur, unannounced)) + if err != nil { + t.Fatalf("Purge: %v", err) + } + if n == 0 { + t.Error("losing the announcement must purge the translated flows") + } + if left := flowsTo(withdrawn) + flowsTo(netip.AddrPortFrom(a(backendA), backendPort)); left != 0 { + t.Errorf("%d translated flows survived the end of the announcement", left) + } } func isTimeout(err error) bool { From fbeca1aa988a2bbb3da7e6e5f9027211ab4900b3 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 18:20:39 +0200 Subject: [PATCH 36/42] docs: correct what the Kyverno guard refuses on CCM Services The RFC said the proposed guard would refuse every later UPDATE of an L4 CCM Service. It does not: Kyverno admits any UPDATE of an object that already violates the rule (validate.allowExistingViolations, true by default). On the lab, with Kyverno 1.18.2, an L4 CCM Service violating through service.cilium.io/type accepted another refused key on UPDATE. The guard does refuse the creation and the migration patch. The narrowed rule makes L4 Services compliant, so it closes that hole for them. It was tested on the lab against a real CCM CREATE and port UPDATE, which the open question now records. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Mattia Eleuteri --- docs/rfc/l4-loadbalancer-mode.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index 63892f6..97c6b60 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -435,8 +435,18 @@ proposed) refuses, on CREATE and UPDATE, any Service carrying `cluster.x-k8s.io/tenant-service-name` that has `spec.externalIPs` or one of a list of MetalLB, external-dns and Cilium annotations, `service.cilium.io/type` included: a tenant must not drive them through the CCM's copy. As proposed, it -would refuse every L4 CCM Service — at creation, and on every later UPDATE, the -CCM's own port updates and the one-shot migration patch included. +refuses an L4 CCM Service at creation, and refuses the one-shot migration patch, +which turns a compliant Service into a violating one. + +It does not refuse the UPDATEs that follow. Kyverno admits any UPDATE of an +object that already violates the rule (`validate.allowExistingViolations`, +`true` by default; verified on the lab with 1.18.2): the CCM's own port updates +pass, but so does an UPDATE that adds another refused key. On the lab, an L4 +CCM Service, violating through its `service.cilium.io/type`, accepted +`service.cilium.io/node` on UPDATE, and another CCM Service already carried a +copied `metallb.io/loadBalancerIPs`. The narrowed rule below closes that hole for +L4 Services: with the label and `ClusterIP`, they are compliant, so every later +UPDATE is validated in full. The two are reconciled by narrowing that one key rather than dropping it: @@ -565,9 +575,11 @@ another tenant, the source IP logged by the backend, MetalLB announcer, and the 9. **Observability**: metrics (sync duration and errors, programmed services, purged flows) and Events on the Service when it is refused. Phase 1 only logs, once per change. -10. **Kyverno guard (7.1)**: the narrowed rule is a sketch; it has to be agreed - with the owners of hikube-gitops !121 and tested on the lab, on CREATE and on - the CCM's UPDATE. +10. **Kyverno guard (7.1)**: the narrowed rule was tested on the lab on + 2026-10-02 (Kyverno 1.18.2): a real CCM CREATE and port UPDATE, the + migration patch, a tenant copy without the label, another + `service.cilium.io/*` key, and the rollback in both orders. It still has to + be agreed with the owners of hikube-gitops !121. 11. **Where the label lives**: `networking.cozystack.io/lb-proxy: cozy-proxy` is a proposal; the maintainers may prefer another key. 12. **Purging the TCP flows of a backend that is still alive — decided.** Aligned From d9e2abda542d857a5ba7200b61f9d7b3c3d10389 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 20:00:32 +0200 Subject: [PATCH 37/42] proxy: draw the masqueraded source port fully at random Under load on the lab (300 connections per second without keep-alive, for two minutes, towards a CCM ingress in L4 mode), 0.06 to 0.1 % of connections took over a second, none under Cilium. On the announcer the backend's SYN-ACK left its veth but never reached ovn0: OVN dropped it, and the client retransmitted. The tuples hit were 100.64.0.x:, reused 29 to 50 s after a previous connection on the same port. By default the kernel keeps the client's source port whenever its own conntrack has the tuple free. It cannot see OVS's, which may still track the previous connection, and every client of a node collapses onto the one masqueraded address. The masquerade of node sources now asks for a fully random port (NF_NAT_RANGE_PROTO_RANDOM_FULLY), which github.com/google/nftables encodes and decodes. The test reads the rule back from the kernel and checks the flag. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/proxy/l4_nft.go | 11 +++++++-- pkg/proxy/l4_nft_test.go | 37 ++++++++++++++++++++++++++++ pkg/proxy/testdata/l4/announcer.nft | 2 +- pkg/proxy/testdata/l4/empty.nft | 2 +- pkg/proxy/testdata/l4/guard-only.nft | 2 +- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/pkg/proxy/l4_nft.go b/pkg/proxy/l4_nft.go index a9a6537..9260522 100644 --- a/pkg/proxy/l4_nft.go +++ b/pkg/proxy/l4_nft.go @@ -112,7 +112,7 @@ func (d *NFTL4Datapath) Teardown() error { // ip daddr V tcp dport P dnat ip addr . port to numgen inc mod N map @backends-... // ip daddr V tcp dport P drop (announced port with no local backend) // chain masq nat postrouting srcnat-5 -// ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade +// ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade fully-random func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { var ids setIDs vips := &nftables.Set{Table: t, ID: ids.next(), Name: "vips", KeyType: nftables.TypeIPAddr} @@ -246,6 +246,13 @@ func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { // // "ct status dnat" restricts it to translated flows, so a node process // talking to a backend pod directly (a kubelet probe) keeps its source. + // + // The source port is drawn fully at random. By default the kernel keeps + // the client's port whenever its own conntrack has the tuple free, but it + // cannot see OVS's: every client of a node collapses onto one masqueraded + // address, and under load the announcer reused a tuple OVS still tracked, + // 29 to 50 s after the previous connection. OVN dropped the backend's + // SYN-ACK and the client retransmitted a second later. conn.AddRule(&nftables.Rule{Table: t, Chain: masq, Exprs: []expr.Any{ &expr.Ct{Register: 1, Key: expr.CtKeySTATUS}, &expr.Bitwise{ @@ -262,7 +269,7 @@ func buildL4Table(conn *nftables.Conn, t *nftables.Table, st l4.State) error { &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: unix.NFT_REG32_01}, &expr.Payload{DestRegister: unix.NFT_REG32_02, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, &expr.Lookup{SourceRegister: unix.NFT_REG32_00, SetName: backends.Name, SetID: backends.ID}, - &expr.Masq{}, + &expr.Masq{FullyRandom: true}, }}) return nil } diff --git a/pkg/proxy/l4_nft_test.go b/pkg/proxy/l4_nft_test.go index 89c5cc5..59ad557 100644 --- a/pkg/proxy/l4_nft_test.go +++ b/pkg/proxy/l4_nft_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/google/nftables" + "github.com/google/nftables/expr" "github.com/vishvananda/netns" v1 "k8s.io/api/core/v1" @@ -258,3 +259,39 @@ func TestL4ChainsCanBeListedByName(t *testing.T) { } } } + +// The masquerade of node sources must pick a fully random source port. By +// default the kernel keeps the client's port whenever its own conntrack has +// the tuple free, and on the lab that made the announcer reuse a +// 100.64.0.x: tuple 29 to 50 s after a previous connection, while OVS +// still tracked it: the backend's SYN-ACK was dropped in OVN and the client +// retransmitted, 0.06 to 0.1 % of connections taking over a second. +func TestL4MasqueradeIsFullyRandom(t *testing.T) { + ns := scratchNetns(t) + if err := datapathIn(ns).Sync(announcerState); err != nil { + t.Fatalf("Sync: %v", err) + } + conn, err := nftables.New(nftables.WithNetNSFd(int(ns))) + if err != nil { + t.Fatalf("nftables.New: %v", err) + } + table := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: L4TableName} + rules, err := conn.GetRules(table, &nftables.Chain{Name: "masq", Table: table}) + if err != nil { + t.Fatalf("GetRules: %v", err) + } + var masqs []*expr.Masq + for _, r := range rules { + for _, e := range r.Exprs { + if m, ok := e.(*expr.Masq); ok { + masqs = append(masqs, m) + } + } + } + if len(masqs) != 1 { + t.Fatalf("want one masquerade in chain masq, got %d", len(masqs)) + } + if !masqs[0].FullyRandom { + t.Errorf("the masquerade keeps the client's source port; want fully-random, got %+v", *masqs[0]) + } +} diff --git a/pkg/proxy/testdata/l4/announcer.nft b/pkg/proxy/testdata/l4/announcer.nft index e7780cb..e2bea9f 100644 --- a/pkg/proxy/testdata/l4/announcer.nft +++ b/pkg/proxy/testdata/l4/announcer.nft @@ -55,6 +55,6 @@ table ip cozy_proxy_l4 { chain masq { type nat hook postrouting priority srcnat - 5; policy accept; - ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade + ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade fully-random } } diff --git a/pkg/proxy/testdata/l4/empty.nft b/pkg/proxy/testdata/l4/empty.nft index f6524b9..0c9dcaa 100644 --- a/pkg/proxy/testdata/l4/empty.nft +++ b/pkg/proxy/testdata/l4/empty.nft @@ -27,6 +27,6 @@ table ip cozy_proxy_l4 { chain masq { type nat hook postrouting priority srcnat - 5; policy accept; - ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade + ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade fully-random } } diff --git a/pkg/proxy/testdata/l4/guard-only.nft b/pkg/proxy/testdata/l4/guard-only.nft index 02f2e02..f9bc347 100644 --- a/pkg/proxy/testdata/l4/guard-only.nft +++ b/pkg/proxy/testdata/l4/guard-only.nft @@ -34,6 +34,6 @@ table ip cozy_proxy_l4 { chain masq { type nat hook postrouting priority srcnat - 5; policy accept; - ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade + ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade fully-random } } From 48b19e7bdae4a142373e92c5bc08054bfdb182a4 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 2 Oct 2026 20:00:46 +0200 Subject: [PATCH 38/42] docs: record the source-port reuse seen under load and its fix The RFC now shows the masquerade with fully-random, and an open question with the lab measurement, the mechanism as far as the kernel code goes, what fully-random changes and what it cannot rule out, and what to look at next if a replay of the load still shows it. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/rfc/l4-loadbalancer-mode.md | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index 97c6b60..f3bd5ff 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -210,7 +210,7 @@ table ip cozy_proxy_l4 { chain masq { type nat hook postrouting priority srcnat - 5; policy accept; - ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade + ct status dnat ip saddr @node_ips ip daddr . meta l4proto . th dport @backends masquerade fully-random } } ``` @@ -587,7 +587,30 @@ another tenant, the source IP logged by the backend, MetalLB announcer, and the go with their frontend (service, port, announcement, including a MetalLB failover). Reason: kube-proxy behaves so, and on the lab an idle client of a withdrawn but live backend hung without a RST until its own timeout. -13. **`google/nftables` limits**: no `typeof` on sets, and no direction on the +13. **Source-port reuse after masquerading node sources (fully-random).** + Measured on the lab: 300 connections per second without keep-alive, for + two minutes, from a pod to a CCM ingress in L4 mode. No error, but 0.06 to + 0.1 % of the connections took over a second, none under Cilium. On the + announcer, the backend VM's SYN-ACK left its veth and never reached `ovn0`: + it was dropped in OVS/OVN, and the client retransmitted. The tuples hit were + `100.64.0.x:`, reused 29 to 50 s after a previous connection on the + same port. + Cause, as far as the code goes: without a flag, the kernel's masquerade + keeps the client's source port whenever *its own* conntrack has the tuple + free (`nf_nat_l4proto_unique_tuple`), and it cannot see OVS's conntrack, + which may still hold the previous connection. Every client of a node + collapses onto one masqueraded address, so the source node's port choices + turn into tuple reuse on the announcer. The suspected drop is the + `ct.inv` match of the kube-ovn ACLs; that part is not confirmed. + Fix in the prototype: `masquerade fully-random` + (`NF_NAT_RANGE_PROTO_RANDOM_FULLY`, a fresh random port per flow; + `random` alone only offsets from a hash). This breaks the correlation with + the source node's choices, but a random port can still land on a tuple OVS + tracks: the residual rate grows with the connection rate and with how long + OVS keeps closed entries. To be measured again under the same load; if it + persists, the next levers are on the OVN side (its conntrack timeouts or + ACLs) or more masquerade addresses per announcer. +14. **`google/nftables` limits**: no `typeof` on sets, and no direction on the typed conntrack keys (3.1). Both are cosmetic for `nft list` today; fixing them upstream would let the masquerade match the original destination. From c3833859ef04ec369c0693b66c103fb737cc64cb Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 5 Oct 2026 08:50:44 +0200 Subject: [PATCH 39/42] main: leave the L4 table alone when told another instance owns it With the L4 mode disabled, every start deleted cozy_proxy_l4. When the L4 mode is tried as a second release next to the platform's cozy-proxy, the platform's instance runs with the mode off and deleted, on each of its starts, the table the other release programs on the same node. Nothing tells the owner: the table came back only with its next forced resync, up to a minute later, and in between the node had neither guard nor translation for the VIPs. A rollout of the platform's release hit every node in turn. --remove-l4-table-when-disabled (chart: l4LoadBalancer.removeTableWhenDisabled) makes that removal explicit. It stays on by default, since it is the rollback path of a cluster running a single release: left off by mistake there, the stale table would keep dropping and translating traffic until removed by hand. The instance next to an L4 release turns it off and leaves the table untouched, as an instance with the VM mode off leaves cozy_proxy. Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Mattia Eleuteri --- charts/cozy-proxy/templates/daemonset.yaml | 6 +++++- charts/cozy-proxy/values.yaml | 7 ++++++- docs/rfc/l4-loadbalancer-mode.md | 8 ++++++++ main.go | 20 +++++++++++++++----- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/charts/cozy-proxy/templates/daemonset.yaml b/charts/cozy-proxy/templates/daemonset.yaml index eea1af3..dc75701 100644 --- a/charts/cozy-proxy/templates/daemonset.yaml +++ b/charts/cozy-proxy/templates/daemonset.yaml @@ -21,11 +21,15 @@ spec: - name: cozy-proxy image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if or .Values.l4LoadBalancer.enabled (not .Values.vmMode.enabled) }} + {{- $keepL4Table := and (not .Values.l4LoadBalancer.enabled) (not .Values.l4LoadBalancer.removeTableWhenDisabled) }} + {{- if or .Values.l4LoadBalancer.enabled (not .Values.vmMode.enabled) $keepL4Table }} args: {{- if .Values.l4LoadBalancer.enabled }} - --enable-l4-loadbalancer {{- end }} + {{- if $keepL4Table }} + - --remove-l4-table-when-disabled=false + {{- end }} {{- if not .Values.vmMode.enabled }} - --enable-vm-mode=false {{- end }} diff --git a/charts/cozy-proxy/values.yaml b/charts/cozy-proxy/values.yaml index 16658d2..90e85cf 100644 --- a/charts/cozy-proxy/values.yaml +++ b/charts/cozy-proxy/values.yaml @@ -14,12 +14,17 @@ rbac: # L4 LoadBalancer mode: takes over from Cilium the public IP of services # labelled networking.cozystack.io/lb-proxy=cozy-proxy that also carry # service.cilium.io/type=ClusterIP. See docs/rfc/l4-loadbalancer-mode.md. -# Disabling it removes the table it programmed on the next start. l4LoadBalancer: enabled: false + # With the mode disabled, remove the table an earlier run programmed on the + # next start: this is how the mode is rolled back. Set it to false on a + # release that runs next to a second release running the L4 mode, or each of + # its starts deletes that release's table. + removeTableWhenDisabled: true # VM mode, for services labelled service.kubernetes.io/service-proxy-name=cozy-proxy. # Turn it off only to run the L4 mode as a second release next to a cozy-proxy # that keeps the VM mode, for instance to try the L4 mode on a live cluster. +# That cozy-proxy then needs l4LoadBalancer.removeTableWhenDisabled=false. vmMode: enabled: true diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index f3bd5ff..efdd9c3 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -353,6 +353,14 @@ commit, so no new flow can be created towards a backend that was just purged. startup, which is the rollback path for the whole mode. - Nothing on the L4 side stops the process: the VM mode runs in the same manager. Without `NODE_NAME`, or while the MetalLB CRD is missing, the mode +- An instance with the mode disabled must not delete the table of another + instance on the same node that runs it, as when the L4 mode is tried as a + second release next to the platform's cozy-proxy: the table would only come + back with the owner's next forced resync, up to a minute later, with neither + guard nor translation for the VIPs in between. That instance runs with + `--remove-l4-table-when-disabled=false` (chart: + `l4LoadBalancer.removeTableWhenDisabled`) and leaves the table untouched, as + an instance with the VM mode off leaves `cozy_proxy`. logs and stays idle. ### 4.5 Announcer changes diff --git a/main.go b/main.go index 184caa0..ce9bbf9 100644 --- a/main.go +++ b/main.go @@ -39,11 +39,16 @@ func main() { var metricsAddr string var enableL4 bool var enableVM bool + var removeL4Table bool flag.StringVar(&probeAddr, "health-probe-bind-address", "0", "The address the probe endpoint binds to. Set to \"0\" to disable.") flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. Set to \"0\" to disable.") flag.BoolVar(&enableL4, "enable-l4-loadbalancer", false, "Run the L4 LoadBalancer mode for services labelled "+l4.ProxyLabel+"="+l4.ProxyLabelValue+ - " (see docs/rfc/l4-loadbalancer-mode.md). When false, a table left by an earlier run is removed.") + " (see docs/rfc/l4-loadbalancer-mode.md).") + flag.BoolVar(&removeL4Table, "remove-l4-table-when-disabled", true, + "With the L4 mode disabled, remove the table left by an earlier run, which is how the mode is rolled back. "+ + "Turn it off on an instance that runs next to another cozy-proxy running the L4 mode: "+ + "each of its starts would delete that instance's table.") flag.BoolVar(&enableVM, "enable-vm-mode", true, "Run the VM mode for services labelled service.kubernetes.io/service-proxy-name=cozy-proxy. "+ "Turn it off only for an instance running the L4 mode next to another cozy-proxy that keeps the VM mode.") @@ -110,7 +115,7 @@ func main() { log.Info("VM mode disabled, the cozy_proxy table is left untouched") } - setupL4(mgr, l4Cfg, enableL4, nodeName) + setupL4(mgr, l4Cfg, enableL4, removeL4Table, nodeName) if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { log.Error(err, "unable to set up health check") @@ -128,12 +133,17 @@ func main() { } } -// setupL4 adds the L4 LoadBalancer mode to the manager, or removes what an -// earlier run of it left when it is disabled. Nothing here may stop the +// setupL4 adds the L4 LoadBalancer mode to the manager. When it is disabled, +// it removes what an earlier run left if removeTable is set, and otherwise +// leaves the table to the instance that owns it. Nothing here may stop the // process: the VM mode runs in the same manager. -func setupL4(mgr ctrl.Manager, cfg *rest.Config, enabled bool, nodeName string) { +func setupL4(mgr ctrl.Manager, cfg *rest.Config, enabled, removeTable bool, nodeName string) { datapath := &proxy.NFTL4Datapath{} if !enabled { + if !removeTable { + log.Info("L4 mode disabled, the " + proxy.L4TableName + " table is left untouched") + return + } if err := datapath.Teardown(); err != nil { log.Error(err, "could not remove the L4 table left by an earlier run") } From 785522e78d238cc2df2b8222dbffe1fa3c5681a6 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 5 Oct 2026 08:50:44 +0200 Subject: [PATCH 40/42] docs: align the RFC's L4 ruleset with what nft lists Section 3.1 named the backend maps after the service, which the code never did: addBackendMap names them after the frontend, VIP, protocol and port, and comments them with the service. The DNAT rule also used a form nft does not print. An operator comparing a node with the RFC would look for maps that do not exist. The example now matches the golden file, and says the map key type is listed as "type 0". Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Mattia Eleuteri --- docs/rfc/l4-loadbalancer-mode.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index efdd9c3..e468966 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -191,8 +191,10 @@ table ip cozy_proxy_l4 { set node_ips { type ipv4_addr } # (backend IP, protocol, backend port) of every local target, announcer only. set backends { type ipv4_addr . inet_proto . inet_service } - # One map per announced service port: round-robin slot -> backend. - map backends-//tcp/ { type integer : ipv4_addr . inet_service } + # One map per announced frontend: round-robin slot -> backend, commented + # with the service's namespace/name. nft lists the key type as "type 0" + # (see addBackendMap). + map backends--tcp- { type integer : ipv4_addr . inet_service } chain guard { type filter hook prerouting priority mangle + 10; policy accept; @@ -203,7 +205,7 @@ table ip cozy_proxy_l4 { chain translate { type nat hook prerouting priority dstnat - 5; policy accept; # announcer only, one rule per (VIP, port) - ip daddr tcp dport dnat ip addr . port to numgen inc mod map @backends-//tcp/ + ip daddr tcp dport dnat ip to numgen inc mod map @backends--tcp- # announcer with no ready local backend for that port ip daddr tcp dport drop } @@ -351,8 +353,6 @@ commit, so no new flow can be created towards a backend that was just purged. - The mode is behind `--enable-l4-loadbalancer` (chart: `l4LoadBalancer.enabled`), off by default. Disabling it deletes the table at startup, which is the rollback path for the whole mode. -- Nothing on the L4 side stops the process: the VM mode runs in the same - manager. Without `NODE_NAME`, or while the MetalLB CRD is missing, the mode - An instance with the mode disabled must not delete the table of another instance on the same node that runs it, as when the L4 mode is tried as a second release next to the platform's cozy-proxy: the table would only come @@ -361,6 +361,8 @@ commit, so no new flow can be created towards a backend that was just purged. `--remove-l4-table-when-disabled=false` (chart: `l4LoadBalancer.removeTableWhenDisabled`) and leaves the table untouched, as an instance with the VM mode off leaves `cozy_proxy`. +- Nothing on the L4 side stops the process: the VM mode runs in the same + manager. Without `NODE_NAME`, or while the MetalLB CRD is missing, the mode logs and stays idle. ### 4.5 Announcer changes From eebf7b871bebcc9061ad02b659f8edb73fafba44 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 5 Oct 2026 08:50:44 +0200 Subject: [PATCH 41/42] docs: fix the README sentence on the announcing node Signed-off-by: Mattia Eleuteri Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Mattia Eleuteri --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b180caf..6a5296e 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,9 @@ spec: externalTrafficPolicy: Local ``` -The node MetalLB (L2) announces the VIP from DNATs new connections to its local -ready backends, masquerading only sources that are node IPs; everything else -addressed to the VIP is dropped. The mode has its own table, `ip +The node from which MetalLB (L2) announces the VIP DNATs new connections to its +local ready backends, masquerading only sources that are node IPs; everything +else addressed to the VIP is dropped. The mode has its own table, `ip cozy_proxy_l4`, and never touches the VM mode's. It is off by default: run with `--enable-l4-loadbalancer` (chart value `l4LoadBalancer.enabled`). Phase 1 handles TCP, IPv4 and `externalTrafficPolicy: Local`. From 612916076a4c9d162c7ac45f15178a19cf6f5b9b Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 6 Oct 2026 16:08:58 +0200 Subject: [PATCH 42/42] docs(rfc): describe the CCM admission guard without an internal reference Section 7.1 and open question 10 pointed to a merge request in a private infrastructure repository. Describe the guard on its own terms instead, and note that Cozystack does not ship one today. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Mattia Eleuteri --- docs/rfc/l4-loadbalancer-mode.md | 33 +++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/docs/rfc/l4-loadbalancer-mode.md b/docs/rfc/l4-loadbalancer-mode.md index e468966..330b040 100644 --- a/docs/rfc/l4-loadbalancer-mode.md +++ b/docs/rfc/l4-loadbalancer-mode.md @@ -405,8 +405,9 @@ Consequences: platform's `infraLabels`. cozy-proxy requires the `networking.cozystack.io/lb-proxy` label (2.5), so the annotation alone never makes it program anything. At worst a tenant copying `service.cilium.io/type: - ClusterIP` makes its own LB dark (Cilium lets go, nobody takes over); the - Kyverno guard of 7.1 refuses such a Service in the first place. + ClusterIP` makes its own LB dark (Cilium lets go, nobody takes over); an + admission guard on CCM Services (7.1), where the platform runs one, refuses + such a Service in the first place. - **What the tenant still controls on an L4 CCM Service**: the ports (the CCM copies them) and `externalTrafficPolicy`. A tenant Service in eTP `Cluster` that the platform labelled anyway would be refused by cozy-proxy and stay @@ -438,15 +439,17 @@ enabled per environment. | Ingress | `packages/extra/ingress/templates/nginx-ingress.yaml` | `controller.service.labels` / `annotations`; eTP is already `Local`. The host ingress (`tenant-root`) with PROXY protocol and ouroboros is last, see Open questions. | | cozy-proxy | `packages/system/cozy-proxy` | bump the vendored chart, enable the mode, extend RBAC (EndpointSlices, Nodes, ServiceL2Status) | -### 7.1 Coexistence with the Kyverno guard on CCM Services +### 7.1 Coexistence with an admission guard on CCM Services -The policy `ccm-lb-tenant-fields-guard` (hikube-gitops !121, `Audit` when -proposed) refuses, on CREATE and UPDATE, any Service carrying -`cluster.x-k8s.io/tenant-service-name` that has `spec.externalIPs` or one of a -list of MetalLB, external-dns and Cilium annotations, `service.cilium.io/type` -included: a tenant must not drive them through the CCM's copy. As proposed, it -refuses an L4 CCM Service at creation, and refuses the one-shot migration patch, -which turns a compliant Service into a violating one. +Because the CCM copies every annotation of the tenant Service onto the infra +Service, a platform may run an admission guard against it. The one considered +here is a Kyverno `ClusterPolicy`: on CREATE and UPDATE, it refuses any Service +carrying `cluster.x-k8s.io/tenant-service-name` that has `spec.externalIPs` or +one of a list of MetalLB, external-dns and Cilium annotations, +`service.cilium.io/type` included, so that a tenant cannot drive them through +the CCM's copy. Such a guard, written that way, refuses an L4 CCM Service at +creation, and refuses the one-shot migration patch, which turns a compliant +Service into a violating one. It does not refuse the UPDATEs that follow. Kyverno admits any UPDATE of an object that already violates the rule (`validate.allowExistingViolations`, @@ -585,11 +588,11 @@ another tenant, the source IP logged by the backend, MetalLB announcer, and the 9. **Observability**: metrics (sync duration and errors, programmed services, purged flows) and Events on the Service when it is refused. Phase 1 only logs, once per change. -10. **Kyverno guard (7.1)**: the narrowed rule was tested on the lab on - 2026-10-02 (Kyverno 1.18.2): a real CCM CREATE and port UPDATE, the - migration patch, a tenant copy without the label, another - `service.cilium.io/*` key, and the rollback in both orders. It still has to - be agreed with the owners of hikube-gitops !121. +10. **Admission guard on CCM Services (7.1)**: the narrowed rule was tested on + the lab on 2026-10-02 (Kyverno 1.18.2): a real CCM CREATE and port UPDATE, + the migration patch, a tenant copy without the label, another + `service.cilium.io/*` key, and the rollback in both orders. Cozystack does + not ship such a guard today: should it, and in which package? 11. **Where the label lives**: `networking.cozystack.io/lb-proxy: cozy-proxy` is a proposal; the maintainers may prefer another key. 12. **Purging the TCP flows of a backend that is still alive — decided.** Aligned