From 3ab8c35117e9468dedaa42d4a2f1e8cf49802016 Mon Sep 17 00:00:00 2001 From: kkk777-7 Date: Sun, 4 Oct 2026 17:49:28 +0900 Subject: [PATCH 1/6] fix(translator): drop backendRefs without ready endpoints when other backends are available Signed-off-by: kkk777-7 --- internal/gatewayapi/route.go | 26 ++ ...ed-invalid-and-valid-backend-refs.out.yaml | 3 +- ...backendrefs-some-without-endpoints.in.yaml | 105 ++++++ ...ackendrefs-some-without-endpoints.out.yaml | 325 ++++++++++++++++++ ...le-backendrefs-some-without-endpoints.yaml | 125 +++++++ ...drefs-some-without-endpoints.clusters.yaml | 127 +++++++ ...refs-some-without-endpoints.endpoints.yaml | 65 ++++ ...refs-some-without-endpoints.listeners.yaml | 43 +++ ...endrefs-some-without-endpoints.routes.yaml | 73 ++++ ...ghted-backend-mixed-valid-and-invalid.yaml | 2 +- test/e2e/tests/weighted_backend.go | 39 ++- 11 files changed, 926 insertions(+), 7 deletions(-) create mode 100644 internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml create mode 100644 internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.routes.yaml diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index dfb4ed71261..5564a6a66eb 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "net" + "slices" "sort" "strconv" "strings" @@ -228,6 +229,14 @@ type routeBackendRefDestination struct { backendClusterKey *BackendClusterKey } +// removeNoEndpointsDestinations returns the destinations without the ones that have no ready endpoints. +// Invalid, dynamic resolver and custom backend destinations are kept as they are handled separately. +func removeNoEndpointsDestinations(destinations []routeBackendRefDestination) []routeBackendRefDestination { + return slices.DeleteFunc(destinations, func(d routeBackendRefDestination) bool { + return !d.ds.Invalid && !d.ds.IsDynamicResolver && !d.ds.IsCustomBackend && len(d.ds.Endpoints) == 0 + }) +} + // httpRouteWithBackendDestinations pairs one rule-match's ir.HTTPRoute with its rule's not-yet-resolved // routeBackendDestinations, deferring the final Settings/BackendClusterRefs split - and thus whether a // merge-eligible backend actually gets a shared cluster - to routeDestinationForListener, once a @@ -359,6 +368,14 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe routeBackendDestinations = append(routeBackendDestinations, backendDest) } + // When some backendRefs in the rule are available, drop the ones without ready endpoints so that traffic + // is redistributed to the available backends instead of returning 503 for their share. + // The BackendsAvailable condition is still set for the dropped backendRefs. + if backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 { + routeBackendDestinations = removeNoEndpointsDestinations(routeBackendDestinations) + backendWeights.NoEndpoints = 0 + } + switch { // return 500 if any filter processing error occurred case processFilterError != nil: @@ -1579,6 +1596,7 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe fmt.Errorf("failed to process route rule %d backendRef %d: %w", ruleIdx, i, err), err.Reason(), )) + backendDest.ds.Invalid = true processDestinationError = err } } @@ -1599,6 +1617,14 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe routeBackendDestinations = append(routeBackendDestinations, backendDest) } + // When some backendRefs in the rule are available, drop the ones without ready endpoints so that traffic + // is redistributed to the available backends instead of returning 503 for their share. + // The BackendsAvailable condition is still set for the dropped backendRefs. + if backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 { + routeBackendDestinations = removeNoEndpointsDestinations(routeBackendDestinations) + backendWeights.NoEndpoints = 0 + } + switch { // return 500 if any filter processing error occurred case processFilterError != nil: diff --git a/internal/gatewayapi/testdata/grpcroute-with-mixed-invalid-and-valid-backend-refs.out.yaml b/internal/gatewayapi/testdata/grpcroute-with-mixed-invalid-and-valid-backend-refs.out.yaml index 408e5064430..7fd84be2d5b 100644 --- a/internal/gatewayapi/testdata/grpcroute-with-mixed-invalid-and-valid-backend-refs.out.yaml +++ b/internal/gatewayapi/testdata/grpcroute-with-mixed-invalid-and-valid-backend-refs.out.yaml @@ -161,7 +161,8 @@ xdsIR: name: grpcroute/default/grpcroute-1/rule/0/backend/0 protocol: GRPC weight: 1 - - name: grpcroute/default/grpcroute-1/rule/0/backend/1 + - invalid: true + name: grpcroute/default/grpcroute-1/rule/0/backend/1 weight: 1 headerMatches: - distinct: false diff --git a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml new file mode 100644 index 00000000000..8762a3ae0fb --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml @@ -0,0 +1,105 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + # the backendRef without endpoints is dropped, service-1 receives all the traffic + - matches: + - path: + type: PathPrefix + value: "/some-without-endpoints" + backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + # the invalid backendRef is kept to return 500 for its share, the backendRef without endpoints is dropped + - matches: + - path: + type: PathPrefix + value: "/mixed-invalid-and-without-endpoints" + backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + - name: service-not-exist + port: 8080 + # all backendRefs have no endpoints, 503 direct response is returned + - matches: + - path: + type: PathPrefix + value: "/all-without-endpoints" + backendRefs: + - name: service-with-no-endpoints + port: 8080 + - name: service-with-no-endpoints + port: 8080 +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: GRPCRoute + metadata: + namespace: default + name: grpcroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + # the backendRef without endpoints is dropped, service-1 receives all the traffic + - matches: + - method: + service: com.example + method: Example + backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 +services: +- apiVersion: v1 + kind: Service + metadata: + namespace: default + name: service-with-no-endpoints + spec: + clusterIP: 1.1.1.1 + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 +endpointSlices: +- apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + namespace: default + name: endpointslice-service-with-no-endpoints + labels: + kubernetes.io/service-name: service-with-no-endpoints + addressType: IPv4 + ports: + - name: http + port: 8080 + protocol: TCP diff --git a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml new file mode 100644 index 00000000000..4644c0a158d --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml @@ -0,0 +1,325 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 2 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: GRPCRoute + metadata: + name: grpcroute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + rules: + - backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + matches: + - method: + method: Example + service: com.example + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: 'Failed to find endpoints: no ready endpoints for the related Service + default/service-with-no-endpoints.' + reason: EndpointsNotFound + status: "False" + type: BackendsAvailable + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: httproute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + rules: + - backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + matches: + - path: + type: PathPrefix + value: /some-without-endpoints + - backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + - name: service-not-exist + port: 8080 + matches: + - path: + type: PathPrefix + value: /mixed-invalid-and-without-endpoints + - backendRefs: + - name: service-with-no-endpoints + port: 8080 + - name: service-with-no-endpoints + port: 8080 + matches: + - path: + type: PathPrefix + value: /all-without-endpoints + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: |- + Failed to find endpoints: no ready endpoints for the related Service default/service-with-no-endpoints. + Failed to find endpoints: no ready endpoints for the related Service default/service-with-no-endpoints. + Failed to find endpoints: no ready endpoints for the related Service default/service-with-no-endpoints. + Failed to find endpoints: no ready endpoints for the related Service default/service-with-no-endpoints. + reason: EndpointsNotFound + status: "False" + type: BackendsAvailable + - lastTransitionTime: null + message: 'Failed to process route rule 1 backendRef 2: service default/service-not-exist + not found.' + reason: BackendNotFound + status: "False" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + ownerReference: + kind: GatewayClass + name: envoy-gateway-class + name: envoy-gateway/gateway-1 + namespace: envoy-gateway-system +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + globalResources: + proxyServiceCluster: + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + settings: + - addressType: IP + endpoints: + - host: 7.6.5.4 + port: 8080 + zone: zone1 + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + protocol: TCP + http: + - address: 0.0.0.0 + externalPort: 80 + grpc: + enableGRPCStats: true + enableGRPCWeb: true + hostnames: + - '*' + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: "8080" + name: grpcroute/default/grpcroute-1/rule/0/backend/0 + protocol: GRPC + weight: 1 + hostname: '*' + isHTTP2: true + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0/match/0/* + pathMatch: + distinct: false + exact: /com.example/Example + name: "" + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: "8080" + name: httproute/default/httproute-1/rule/1/backend/0 + protocol: HTTP + weight: 1 + - invalid: true + name: httproute/default/httproute-1/rule/1/backend/2 + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/match/0/* + pathMatch: + distinct: false + name: "" + prefix: /mixed-invalid-and-without-endpoints + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: "8080" + name: httproute/default/httproute-1/rule/0/backend/0 + protocol: HTTP + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/* + pathMatch: + distinct: false + name: "" + prefix: /some-without-endpoints + - directResponse: + statusCode: 503 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/2/match/0/* + pathMatch: + distinct: false + name: "" + prefix: /all-without-endpoints + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.yaml new file mode 100644 index 00000000000..4a6e3b07ff0 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.yaml @@ -0,0 +1,125 @@ +http: +- address: 0.0.0.0 + externalPort: 80 + grpc: + enableGRPCStats: true + enableGRPCWeb: true + hostnames: + - '*' + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: '8080' + name: grpcroute/default/grpcroute-1/rule/0/backend/0 + protocol: GRPC + weight: 1 + hostname: '*' + isHTTP2: true + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0/match/0/* + pathMatch: + distinct: false + exact: /com.example/Example + name: '' + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: '8080' + name: httproute/default/httproute-1/rule/1/backend/0 + protocol: HTTP + weight: 1 + - invalid: true + name: httproute/default/httproute-1/rule/1/backend/2 + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/match/0/* + pathMatch: + distinct: false + name: '' + prefix: /mixed-invalid-and-without-endpoints + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: '8080' + name: httproute/default/httproute-1/rule/0/backend/0 + protocol: HTTP + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/* + pathMatch: + distinct: false + name: '' + prefix: /some-without-endpoints + - directResponse: + statusCode: 503 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/2/match/0/* + pathMatch: + distinct: false + name: '' + prefix: /all-without-endpoints diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.clusters.yaml new file mode 100644 index 00000000000..721ede94b10 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.clusters.yaml @@ -0,0 +1,127 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: grpcroute/default/grpcroute-1/rule/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0 + perConnectionBufferLimitBytes: 32768 + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: httproute/default/httproute-1/rule/1/backend/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/backend/0 + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: httproute/default/httproute-1/rule/1/backend/2 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/backend/2 + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: httproute/default/httproute-1/rule/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0 + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.endpoints.yaml new file mode 100644 index 00000000000..69521a04791 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.endpoints.yaml @@ -0,0 +1,65 @@ +- clusterName: grpcroute/default/grpcroute-1/rule/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 7.7.7.7 + portValue: 8080 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: grpcroute/default/grpcroute-1/rule/0/backend/0 + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: Service + name: service-1 + namespace: default + sectionName: "8080" +- clusterName: httproute/default/httproute-1/rule/1/backend/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 7.7.7.7 + portValue: 8080 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: httproute/default/httproute-1/rule/1/backend/0 + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: Service + name: service-1 + namespace: default + sectionName: "8080" +- clusterName: httproute/default/httproute-1/rule/1/backend/2 + endpoints: + - loadBalancingWeight: 1 + locality: + region: httproute/default/httproute-1/rule/1/backend/2 +- clusterName: httproute/default/httproute-1/rule/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 7.7.7.7 + portValue: 8080 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: httproute/default/httproute-1/rule/0/backend/0 + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: Service + name: service-1 + namespace: default + sectionName: "8080" diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.listeners.yaml new file mode 100644 index 00000000000..1bf3c32ffbb --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.listeners.yaml @@ -0,0 +1,43 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.grpc_web + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb + - name: envoy.filters.http.grpc_stats + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.grpc_stats.v3.FilterConfig + emitFilterState: true + statsForAllMethods: true + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: envoy-gateway/gateway-1/http + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: envoy-gateway/gateway-1/http + maxConnectionsToAcceptPerSocketEvent: 1 + name: envoy-gateway/gateway-1/http + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.routes.yaml new file mode 100644 index 00000000000..9a1da48ef54 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.routes.yaml @@ -0,0 +1,73 @@ +- ignorePortInHostMatching: true + name: envoy-gateway/gateway-1/http + virtualHosts: + - domains: + - '*' + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http/* + routes: + - match: + path: /com.example/Example + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0/match/0/* + route: + cluster: grpcroute/default/grpcroute-1/rule/0 + - match: + pathSeparatedPrefix: /mixed-invalid-and-without-endpoints + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/match/0/* + route: + clusterNotFoundResponseCode: INTERNAL_SERVER_ERROR + upgradeConfigs: + - upgradeType: websocket + weightedClusters: + clusters: + - name: invalid-backend-cluster + weight: 1 + - name: httproute/default/httproute-1/rule/1/backend/0 + weight: 1 + - match: + pathSeparatedPrefix: /some-without-endpoints + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/* + route: + cluster: httproute/default/httproute-1/rule/0 + upgradeConfigs: + - upgradeType: websocket + - directResponse: + status: 503 + match: + pathSeparatedPrefix: /all-without-endpoints + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/2/match/0/* diff --git a/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml b/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml index c02adeea395..93cc543feea 100644 --- a/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml +++ b/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml @@ -26,7 +26,7 @@ spec: - name: infra-backend-v1 port: 8080 weight: 9 - - name: infra-backend-without-endpoints # should return 503 for empty cluster + - name: infra-backend-without-endpoints # should be dropped, infra-backend-v1 receives all the traffic port: 8080 weight: 1 --- diff --git a/test/e2e/tests/weighted_backend.go b/test/e2e/tests/weighted_backend.go index 4bf5e59ea69..d20e286e67d 100644 --- a/test/e2e/tests/weighted_backend.go +++ b/test/e2e/tests/weighted_backend.go @@ -154,11 +154,6 @@ func testMixedValidAndInvalid(t *testing.T, suite *suite.ConformanceTestSuite) { path: "/mixed-valid-and-invalid", failureCode: 500, }, - { - name: "MixedValidAndNoEndpoints", - path: "/mixed-valid-and-no-endpoints", - failureCode: 503, - }, } for _, scenario := range scenarios { @@ -166,6 +161,11 @@ func testMixedValidAndInvalid(t *testing.T, suite *suite.ConformanceTestSuite) { runMixedValidAndInvalidScenario(t, suite, gwAddr, scenario.path, scenario.failureCode) }) } + + // The backendRef without endpoints is dropped, so all the requests are routed to the valid backend. + t.Run("MixedValidAndNoEndpoints", func(t *testing.T) { + runMixedValidAndNoEndpointsScenario(t, suite, gwAddr, "/mixed-valid-and-no-endpoints") + }) } const ( @@ -228,3 +228,32 @@ func runMixedValidAndInvalidScenario(t *testing.T, suite *suite.ConformanceTestS t.Logf("success count for %s is %d, failure count for %s is %d", path, successCount, path, failureCount) } + +func runMixedValidAndNoEndpointsScenario(t *testing.T, suite *suite.ConformanceTestSuite, gwAddr, path string) { + t.Helper() + + expected := http.ExpectedResponse{ + Request: http.Request{ + Path: path, + }, + Response: http.Response{ + StatusCodes: []int{200}, + }, + Namespace: ConformanceInfraNamespace, + } + + // Make sure the valid(response 200) backend are ready. + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expected) + + req := http.MakeRequest(t, &expected, gwAddr, "HTTP", "http") + for i := 0; i < mixedValidAndInvalidRequests; i++ { + _, response, err := suite.RoundTripper.CaptureRoundTrip(req) + if err != nil { + t.Errorf("failed to get expected response: %v", err) + continue + } + if response.StatusCode != 200 { + t.Errorf("unexpected status code %d for %s", response.StatusCode, path) + } + } +} From df32bd78fda5db5e8b2c045a8c77ac726784aab9 Mon Sep 17 00:00:00 2001 From: kkk777-7 Date: Sun, 4 Oct 2026 17:50:51 +0900 Subject: [PATCH 2/6] docs: add release notes for #10180 Signed-off-by: kkk777-7 --- .../10180-drop-backendrefs-without-ready-endpoints.md | 1 + .../bug_fixes/10180-grpcroute-invalid-backend-response-code.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md create mode 100644 release-notes/current/bug_fixes/10180-grpcroute-invalid-backend-response-code.md diff --git a/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md b/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md new file mode 100644 index 00000000000..02346c080b8 --- /dev/null +++ b/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md @@ -0,0 +1 @@ +HTTPRoute and GRPCRoute rules with multiple backendRefs no longer return 503 for the share of traffic routed to backendRefs without ready endpoints when at least one other backendRef in the same rule is available; those backendRefs are excluded from the rule so the remaining backends receive all the traffic per their relative weights. The `BackendsAvailable=False` route condition is still reported, and a 503 is still returned when all backendRefs in the rule have no ready endpoints. As a result, rules mixing invalid backendRefs and backendRefs without ready endpoints now return 500 for the invalid share instead of 503. diff --git a/release-notes/current/bug_fixes/10180-grpcroute-invalid-backend-response-code.md b/release-notes/current/bug_fixes/10180-grpcroute-invalid-backend-response-code.md new file mode 100644 index 00000000000..6a33faec422 --- /dev/null +++ b/release-notes/current/bug_fixes/10180-grpcroute-invalid-backend-response-code.md @@ -0,0 +1 @@ +Fixed GRPCRoute returning 503 instead of 500 for the share of traffic routed to invalid backendRefs (e.g. a non-existent Service) when the rule also has valid backendRefs. From af32c80dba7ebfe8cbeb8c04d30e91cdb3c6aee5 Mon Sep 17 00:00:00 2001 From: kkk777-7 Date: Sat, 10 Oct 2026 11:33:00 +0900 Subject: [PATCH 3/6] fix: address review comments on dropping backendRefs without endpoints Signed-off-by: kkk777-7 --- internal/gatewayapi/route.go | 8 +++- ...backendrefs-some-without-endpoints.in.yaml | 2 +- ...ackendrefs-some-without-endpoints.out.yaml | 2 + internal/xds/translator/route.go | 11 +++-- .../http-route-weighted-invalid-backend.yaml | 14 ++++++ ...ute-weighted-invalid-backend.clusters.yaml | 46 +++++++++++++++++++ ...te-weighted-invalid-backend.endpoints.yaml | 17 +++++++ ...route-weighted-invalid-backend.routes.yaml | 10 ++++ ...rop-backendrefs-without-ready-endpoints.md | 2 +- ...grpcroute-invalid-backend-response-code.md | 1 - 10 files changed, 104 insertions(+), 9 deletions(-) delete mode 100644 release-notes/current/bug_fixes/10180-grpcroute-invalid-backend-response-code.md diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index 5564a6a66eb..3ffc1a69b45 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -371,7 +371,9 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe // When some backendRefs in the rule are available, drop the ones without ready endpoints so that traffic // is redistributed to the available backends instead of returning 503 for their share. // The BackendsAvailable condition is still set for the dropped backendRefs. - if backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 { + // Skip this when the rule has invalid backendRefs, as the dropped share would also go to the invalid ones + // and change their configured fraction. + if backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 && backendWeights.Invalid == 0 { routeBackendDestinations = removeNoEndpointsDestinations(routeBackendDestinations) backendWeights.NoEndpoints = 0 } @@ -1620,7 +1622,9 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe // When some backendRefs in the rule are available, drop the ones without ready endpoints so that traffic // is redistributed to the available backends instead of returning 503 for their share. // The BackendsAvailable condition is still set for the dropped backendRefs. - if backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 { + // Skip this when the rule has invalid backendRefs, as the dropped share would also go to the invalid ones + // and change their configured fraction. + if backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 && backendWeights.Invalid == 0 { routeBackendDestinations = removeNoEndpointsDestinations(routeBackendDestinations) backendWeights.NoEndpoints = 0 } diff --git a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml index 8762a3ae0fb..ae51fada900 100644 --- a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml +++ b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml @@ -34,7 +34,7 @@ httpRoutes: port: 8080 - name: service-with-no-endpoints port: 8080 - # the invalid backendRef is kept to return 500 for its share, the backendRef without endpoints is dropped + # the backendRef without endpoints is kept to preserve the invalid fraction, the invalid and no endpoints share returns 503 - matches: - path: type: PathPrefix diff --git a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml index 4644c0a158d..570782d6ceb 100644 --- a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml @@ -261,6 +261,8 @@ xdsIR: name: httproute/default/httproute-1/rule/1/backend/0 protocol: HTTP weight: 1 + - name: httproute/default/httproute-1/rule/1/backend/1 + weight: 1 - invalid: true name: httproute/default/httproute-1/rule/1/backend/2 weight: 1 diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index bdc26bce056..2360f58e54b 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -337,7 +337,8 @@ func buildXdsRouteAction(route *ir.HTTPRoute, backendIndex backendClusterIndex) // Weighted routing is needed whenever there's more than one cluster to route between. if route.NeedsClusterPerSetting() || clusterCount > 1 { - return buildXdsWeightedRouteAction(route.Destination, backendIndex) + // IsHTTP2 is only set for GRPCRoute. + return buildXdsWeightedRouteAction(route.Destination, backendIndex, route.IsHTTP2) } // Defaults to the destination's own name; a single backend cluster overrides it. @@ -352,7 +353,7 @@ func buildXdsRouteAction(route *ir.HTTPRoute, backendIndex backendClusterIndex) } } -func buildXdsWeightedRouteAction(destination *ir.RouteDestination, backendIndex backendClusterIndex) *routev3.RouteAction { +func buildXdsWeightedRouteAction(destination *ir.RouteDestination, backendIndex backendClusterIndex, isGRPC bool) *routev3.RouteAction { weightedClusters := make([]*routev3.WeightedCluster_ClusterWeight, 0) backendWeights := destination.ToBackendWeights() @@ -398,8 +399,10 @@ func buildXdsWeightedRouteAction(destination *ir.RouteDestination, backendIndex // 503 status code should be returned for Services without ready endpoints // Reference: https://gateway-api.sigs.k8s.io/reference/api-spec/main/spec/#httprouterule clusterNotFoundResponseCode := routev3.RouteAction_INTERNAL_SERVER_ERROR - // Envoy can't handle mixed 500 and 503 responses, so we use 503 when both invalid and empty - if backendWeights.NoEndpoints > 0 { + // Envoy can't handle mixed 500 and 503 responses, so we use 503 when both invalid and empty. + // For GRPCRoute, invalid GRPCBackendRefs must receive an UNAVAILABLE status, which Envoy maps from 503. + // Reference: https://gateway-api.sigs.k8s.io/reference/api-spec/main/spec/#grpcrouterule + if backendWeights.NoEndpoints > 0 || isGRPC { clusterNotFoundResponseCode = routev3.RouteAction_SERVICE_UNAVAILABLE } return &routev3.RouteAction{ diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-invalid-backend.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-invalid-backend.yaml index be1a5a6223f..8acccf69909 100644 --- a/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-invalid-backend.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-invalid-backend.yaml @@ -48,3 +48,17 @@ http: invalid: true - weight: 1 name: "third-route-dest/backend/2" # clusterNotFoundResponseCode should be 503 for mixed empty and invalid cluster + - name: "fourth-route" + hostname: "*" + isHTTP2: true + destination: + name: "fourth-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + weight: 1 + name: "fourth-route-dest/backend/0" + - weight: 1 + name: "fourth-route-dest/backend/1" + invalid: true # clusterNotFoundResponseCode should be 503 for invalid cluster of GRPCRoute diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml index aa81c7ec74a..17415403def 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml @@ -159,3 +159,49 @@ name: third-route-dest/backend/2 perConnectionBufferLimitBytes: 32768 type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: fourth-route-dest/backend/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: fourth-route-dest/backend/0 + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: fourth-route-dest/backend/1 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: fourth-route-dest/backend/1 + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml index 39278f337cc..80c139f19a1 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml @@ -54,3 +54,20 @@ - loadBalancingWeight: 1 locality: region: third-route-dest/backend/2 +- clusterName: fourth-route-dest/backend/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: fourth-route-dest/backend/0 +- clusterName: fourth-route-dest/backend/1 + endpoints: + - loadBalancingWeight: 1 + locality: + region: fourth-route-dest/backend/1 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml index 10b361451ab..e37e2270dcd 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml @@ -42,3 +42,13 @@ weight: 2 - name: third-route-dest/backend/0 weight: 1 + - match: + prefix: / + name: fourth-route + route: + weightedClusters: + clusters: + - name: invalid-backend-cluster + weight: 1 + - name: fourth-route-dest/backend/0 + weight: 1 diff --git a/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md b/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md index 02346c080b8..477ddfb02f4 100644 --- a/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md +++ b/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md @@ -1 +1 @@ -HTTPRoute and GRPCRoute rules with multiple backendRefs no longer return 503 for the share of traffic routed to backendRefs without ready endpoints when at least one other backendRef in the same rule is available; those backendRefs are excluded from the rule so the remaining backends receive all the traffic per their relative weights. The `BackendsAvailable=False` route condition is still reported, and a 503 is still returned when all backendRefs in the rule have no ready endpoints. As a result, rules mixing invalid backendRefs and backendRefs without ready endpoints now return 500 for the invalid share instead of 503. +HTTPRoute and GRPCRoute rules with multiple backendRefs no longer return 503 for the share of traffic routed to backendRefs without ready endpoints when at least one other backendRef in the same rule is available; those backendRefs are excluded from the rule so the remaining backends receive all the traffic per their relative weights. The `BackendsAvailable=False` route condition is still reported, and a 503 is still returned when all backendRefs in the rule have no ready endpoints. Rules that also have invalid backendRefs are unchanged, so the configured share of the invalid backendRefs is preserved. diff --git a/release-notes/current/bug_fixes/10180-grpcroute-invalid-backend-response-code.md b/release-notes/current/bug_fixes/10180-grpcroute-invalid-backend-response-code.md deleted file mode 100644 index 6a33faec422..00000000000 --- a/release-notes/current/bug_fixes/10180-grpcroute-invalid-backend-response-code.md +++ /dev/null @@ -1 +0,0 @@ -Fixed GRPCRoute returning 503 instead of 500 for the share of traffic routed to invalid backendRefs (e.g. a non-existent Service) when the rule also has valid backendRefs. From 3c1d56d7dee2f4169ed5f137da45e09d10e1c9bf Mon Sep 17 00:00:00 2001 From: kkk777-7 Date: Fri, 16 Oct 2026 01:58:27 +0900 Subject: [PATCH 4/6] generate testdata Signed-off-by: kkk777-7 --- ...tiple-backendrefs-some-without-endpoints.out.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml index 570782d6ceb..4e22493d511 100644 --- a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml @@ -129,6 +129,12 @@ httpRoutes: reason: Accepted status: "True" type: Accepted + - lastTransitionTime: null + message: 'Failed to process route rule 1 backendRef 2: service default/service-not-exist + not found.' + reason: BackendNotFound + status: "False" + type: ResolvedRefs - lastTransitionTime: null message: |- Failed to find endpoints: no ready endpoints for the related Service default/service-with-no-endpoints. @@ -138,12 +144,6 @@ httpRoutes: reason: EndpointsNotFound status: "False" type: BackendsAvailable - - lastTransitionTime: null - message: 'Failed to process route rule 1 backendRef 2: service default/service-not-exist - not found.' - reason: BackendNotFound - status: "False" - type: ResolvedRefs controllerName: gateway.envoyproxy.io/gatewayclass-controller parentRef: name: gateway-1 From d61d02f0f831990f0cf167dbfefbe3ff36ea2bf5 Mon Sep 17 00:00:00 2001 From: kkk777-7 Date: Mon, 19 Oct 2026 19:11:58 +0900 Subject: [PATCH 5/6] feat(translator): guard dropping backendRefs without endpoints behind a runtime flag Signed-off-by: kkk777-7 --- api/v1alpha1/envoygateway_helpers.go | 7 ++++--- api/v1alpha1/envoygateway_types.go | 10 +++++++++- internal/gatewayapi/route.go | 16 ++++++++++------ internal/gatewayapi/runner/runner.go | 1 + internal/gatewayapi/translator.go | 4 ++++ internal/gatewayapi/translator_test.go | 8 ++++++++ ...0-drop-backendrefs-without-ready-endpoints.md | 1 - ...0-drop-backendrefs-without-ready-endpoints.md | 1 + site/content/en/latest/api/extension_types.md | 1 + .../xds-name-scheme-v2.yaml | 1 + test/config/helm/xds-name-scheme-v2.yaml | 1 + ...weighted-backend-mixed-valid-and-invalid.yaml | 2 +- test/e2e/tests/utils.go | 7 +++++++ test/e2e/tests/weighted_backend.go | 8 ++++++-- 14 files changed, 54 insertions(+), 14 deletions(-) delete mode 100644 release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md create mode 100644 release-notes/current/new_features/10180-drop-backendrefs-without-ready-endpoints.md diff --git a/api/v1alpha1/envoygateway_helpers.go b/api/v1alpha1/envoygateway_helpers.go index 661a7961330..66b097f9d7f 100644 --- a/api/v1alpha1/envoygateway_helpers.go +++ b/api/v1alpha1/envoygateway_helpers.go @@ -171,9 +171,10 @@ func (e *EnvoyGateway) GetEnvoyProxyDefaultSpec() *EnvoyProxySpec { // defaultRuntimeFlags are the default runtime flags for Envoy Gateway. var defaultRuntimeFlags = map[RuntimeFlag]bool{ - XDSNameSchemeV2: false, - EndpointSliceIndex: true, - EnvoyProxyPatch: true, + XDSNameSchemeV2: false, + EndpointSliceIndex: true, + EnvoyProxyPatch: true, + DropBackendRefsWithoutEndpoints: false, } // IsEnabled checks if an experimental Gateway API is enabled in the EnvoyGateway configuration. diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index b8f9a3c7b9b..2d062076a36 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -170,7 +170,7 @@ type GatewayAPISettings struct { // RuntimeFlag defines a runtime flag used to guard breaking changes or risky experimental features in new Envoy Gateway releases. // A runtime flag may be enabled or disabled by default and can be toggled through the EnvoyGateway resource. // +enum -// +kubebuilder:validation:Enum=XDSNameSchemeV2;EndpointSliceIndex;PerResourceSystemCASecret;EnvoyProxyPatch +// +kubebuilder:validation:Enum=XDSNameSchemeV2;EndpointSliceIndex;PerResourceSystemCASecret;EnvoyProxyPatch;DropBackendRefsWithoutEndpoints type RuntimeFlag string const ( @@ -199,6 +199,14 @@ const ( // Gateway's more privileged ServiceAccount; disable this flag in multi-tenant clusters // where tenants can author their own EnvoyProxy resources. EnvoyProxyPatch RuntimeFlag = "EnvoyProxyPatch" + + // DropBackendRefsWithoutEndpoints excludes backendRefs without ready endpoints from an HTTPRoute + // or GRPCRoute rule when at least one other backendRef in the same rule is available, so the + // remaining backends receive all the traffic per their relative weights instead of returning 503 + // for the share of the excluded backendRefs. + // Rules that also have invalid backendRefs are unchanged. + // Disabled by default. + DropBackendRefsWithoutEndpoints RuntimeFlag = "DropBackendRefsWithoutEndpoints" ) // RuntimeFlags provide a mechanism to guard breaking changes or risky experimental features in new Envoy Gateway releases. diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index 2101f51a555..53eb5a5bd32 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -368,12 +368,14 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe routeBackendDestinations = append(routeBackendDestinations, backendDest) } - // When some backendRefs in the rule are available, drop the ones without ready endpoints so that traffic - // is redistributed to the available backends instead of returning 503 for their share. + // When DropBackendRefsWithoutEndpoints is enabled and some backendRefs in the rule are available, drop the + // ones without ready endpoints so that traffic is redistributed to the available backends instead of + // returning 503 for their share. // The BackendsAvailable condition is still set for the dropped backendRefs. // Skip this when the rule has invalid backendRefs, as the dropped share would also go to the invalid ones // and change their configured fraction. - if backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 && backendWeights.Invalid == 0 { + if t.DropBackendRefsWithoutEndpoints && + backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 && backendWeights.Invalid == 0 { routeBackendDestinations = removeNoEndpointsDestinations(routeBackendDestinations) backendWeights.NoEndpoints = 0 } @@ -1619,12 +1621,14 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe routeBackendDestinations = append(routeBackendDestinations, backendDest) } - // When some backendRefs in the rule are available, drop the ones without ready endpoints so that traffic - // is redistributed to the available backends instead of returning 503 for their share. + // When DropBackendRefsWithoutEndpoints is enabled and some backendRefs in the rule are available, drop the + // ones without ready endpoints so that traffic is redistributed to the available backends instead of + // returning 503 for their share. // The BackendsAvailable condition is still set for the dropped backendRefs. // Skip this when the rule has invalid backendRefs, as the dropped share would also go to the invalid ones // and change their configured fraction. - if backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 && backendWeights.Invalid == 0 { + if t.DropBackendRefsWithoutEndpoints && + backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 && backendWeights.Invalid == 0 { routeBackendDestinations = removeNoEndpointsDestinations(routeBackendDestinations) backendWeights.NoEndpoints = 0 } diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index bfcd390f8d0..9465bb407f7 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -330,6 +330,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re MergeGateways: gatewayapi.IsMergeGatewaysEnabled(resources), MergeBackends: gatewayapi.ResolveMergeBackendsConfig(resources), PerResourceSystemCASecret: r.EnvoyGateway.RuntimeFlags.IsEnabled(egv1a1.PerResourceSystemCASecret), + DropBackendRefsWithoutEndpoints: r.EnvoyGateway.RuntimeFlags.IsEnabled(egv1a1.DropBackendRefsWithoutEndpoints), WasmCache: r.wasmCache, RunningOnHost: r.EnvoyGateway.Provider != nil && r.EnvoyGateway.Provider.IsRunningOnHost(), InfraRemotelyManaged: r.EnvoyGateway.Provider != nil && r.EnvoyGateway.Provider.IsInfraManagedRemotely(), diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 979465facdb..cf07401869a 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -140,6 +140,10 @@ type Translator struct { // sharing a single system_ca_certificates secret. Disabled by default (shared secret used). PerResourceSystemCASecret bool + // DropBackendRefsWithoutEndpoints excludes backendRefs without ready endpoints from a route rule + // when at least one other backendRef in the same rule is available. Disabled by default. + DropBackendRefsWithoutEndpoints bool + // GatewayNamespaceMode is true if controller uses gateway namespace mode for infra deployments. GatewayNamespaceMode bool diff --git a/internal/gatewayapi/translator_test.go b/internal/gatewayapi/translator_test.go index dcc5e557089..d52bead08d5 100644 --- a/internal/gatewayapi/translator_test.go +++ b/internal/gatewayapi/translator_test.go @@ -58,6 +58,7 @@ func TestTranslate(t *testing.T) { SDSEnabled bool PerResourceSystemCASecret bool EnvoyProxyPatchDisabled bool + DropBackendRefsWithoutEndpoints bool }{ { name: "envoypatchpolicy-invalid-feature-disabled", @@ -139,6 +140,10 @@ func TestTranslate(t *testing.T) { name: "envoyproxy-patch-disabled", EnvoyProxyPatchDisabled: true, }, + { + name: "httproute-with-multiple-backendrefs-some-without-endpoints", + DropBackendRefsWithoutEndpoints: true, + }, } inputFiles, err := filepath.Glob(filepath.Join("testdata", "*.in.yaml")) @@ -166,6 +171,7 @@ func TestTranslate(t *testing.T) { sdsEnabled := false perResourceSystemCASecret := false envoyProxyPatchDisabled := false + dropBackendRefsWithoutEndpoints := false for _, config := range testCasesConfig { if config.name == strings.Split(filepath.Base(inputFile), ".")[0] { @@ -177,6 +183,7 @@ func TestTranslate(t *testing.T) { sdsEnabled = config.SDSEnabled perResourceSystemCASecret = config.PerResourceSystemCASecret envoyProxyPatchDisabled = config.EnvoyProxyPatchDisabled + dropBackendRefsWithoutEndpoints = config.DropBackendRefsWithoutEndpoints } } @@ -188,6 +195,7 @@ func TestTranslate(t *testing.T) { BackendEnabled: backendEnabled, SDSSecretRefEnabled: sdsEnabled, PerResourceSystemCASecret: perResourceSystemCASecret, + DropBackendRefsWithoutEndpoints: dropBackendRefsWithoutEndpoints, ControllerNamespace: "envoy-gateway-system", MergeGateways: IsMergeGatewaysEnabled(resources), MergeBackends: ResolveMergeBackendsConfig(resources), diff --git a/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md b/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md deleted file mode 100644 index 477ddfb02f4..00000000000 --- a/release-notes/current/breaking_changes/10180-drop-backendrefs-without-ready-endpoints.md +++ /dev/null @@ -1 +0,0 @@ -HTTPRoute and GRPCRoute rules with multiple backendRefs no longer return 503 for the share of traffic routed to backendRefs without ready endpoints when at least one other backendRef in the same rule is available; those backendRefs are excluded from the rule so the remaining backends receive all the traffic per their relative weights. The `BackendsAvailable=False` route condition is still reported, and a 503 is still returned when all backendRefs in the rule have no ready endpoints. Rules that also have invalid backendRefs are unchanged, so the configured share of the invalid backendRefs is preserved. diff --git a/release-notes/current/new_features/10180-drop-backendrefs-without-ready-endpoints.md b/release-notes/current/new_features/10180-drop-backendrefs-without-ready-endpoints.md new file mode 100644 index 00000000000..27a6e96c5af --- /dev/null +++ b/release-notes/current/new_features/10180-drop-backendrefs-without-ready-endpoints.md @@ -0,0 +1 @@ +Added the `DropBackendRefsWithoutEndpoints` runtime flag (disabled by default). When enabled, HTTPRoute and GRPCRoute rules with multiple backendRefs no longer return 503 for the share of traffic routed to backendRefs without ready endpoints when at least one other backendRef in the same rule is available; those backendRefs are excluded from the rule so the remaining backends receive all the traffic per their relative weights. The `BackendsAvailable=False` route condition is still reported, and a 503 is still returned when all backendRefs in the rule have no ready endpoints. Rules that also have invalid backendRefs are unchanged, so the configured share of the invalid backendRefs is preserved. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 7fb68d6391f..53e1fbceb4f 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -6326,6 +6326,7 @@ _Appears in:_ | `EndpointSliceIndex` | EndpointSliceIndex indicates that field indexes are used to look up EndpointSlices by backend.
It is enabled by default to reduce CPU usage for EndpointSlice lookups in large clusters.
If the additional controller memory usage for the indexes becomes a concern,
consider disabling this flag.
| | `PerResourceSystemCASecret` | PerResourceSystemCASecret restores the pre-1.x behavior of emitting one SDS secret per
BackendTLSPolicy or Backend resource that uses WellKnownCACertificates: System, instead
of sharing a single system_ca_certificates secret across all of them.
Disabled by default (i.e. the shared secret is used). Enable this flag to opt out during
upgrades — Envoy must warm the new system_ca_certificates secret before clusters can use
it, which may cause a brief disruption to new connections on first enable.
| | `EnvoyProxyPatch` | EnvoyProxyPatch enables applying the Kubernetes resource `patch` fields configured on
EnvoyProxy's Kubernetes provider settings. It is enabled by default to preserve
pre-existing behavior. Because EnvoyProxy is commonly namespace-scoped and
tenant-authored, a patch may grant arbitrary access to resources applied by Envoy
Gateway's more privileged ServiceAccount; disable this flag in multi-tenant clusters
where tenants can author their own EnvoyProxy resources.
| +| `DropBackendRefsWithoutEndpoints` | DropBackendRefsWithoutEndpoints excludes backendRefs without ready endpoints from an HTTPRoute
or GRPCRoute rule when at least one other backendRef in the same rule is available, so the
remaining backends receive all the traffic per their relative weights instead of returning 503
for the share of the excluded backendRefs.
Rules that also have invalid backendRefs are unchanged.
Disabled by default.
| #### RuntimeFlags diff --git a/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml b/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml index b10cbd7e70c..6c843965ab4 100644 --- a/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml +++ b/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml @@ -34,6 +34,7 @@ data: enabled: - XDSNameSchemeV2 - EnvoyProxyPatch + - DropBackendRefsWithoutEndpoints telemetry: traces: sink: diff --git a/test/config/helm/xds-name-scheme-v2.yaml b/test/config/helm/xds-name-scheme-v2.yaml index 290d342c3e9..62f8c2101d4 100644 --- a/test/config/helm/xds-name-scheme-v2.yaml +++ b/test/config/helm/xds-name-scheme-v2.yaml @@ -3,3 +3,4 @@ config: runtimeFlags: enabled: - XDSNameSchemeV2 + - DropBackendRefsWithoutEndpoints diff --git a/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml b/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml index 93cc543feea..7190e5ade78 100644 --- a/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml +++ b/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml @@ -26,7 +26,7 @@ spec: - name: infra-backend-v1 port: 8080 weight: 9 - - name: infra-backend-without-endpoints # should be dropped, infra-backend-v1 receives all the traffic + - name: infra-backend-without-endpoints # should return 503 for empty cluster, or be dropped when DropBackendRefsWithoutEndpoints is enabled port: 8080 weight: 1 --- diff --git a/test/e2e/tests/utils.go b/test/e2e/tests/utils.go index 6b36b9987bc..5f01a137c88 100644 --- a/test/e2e/tests/utils.go +++ b/test/e2e/tests/utils.go @@ -1234,6 +1234,13 @@ func XDSNameSchemeV2() bool { return DeployProfile == "xds-name-scheme-v2" } +// DropBackendRefsWithoutEndpoints returns true if the DropBackendRefsWithoutEndpoints runtime flag is enabled +// in the deploy profile. +// TODO: remove this after the feature flag is removed. +func DropBackendRefsWithoutEndpoints() bool { + return DeployProfile == "xds-name-scheme-v2" +} + func GetGatewayResourceNamespace() string { if IsGatewayNamespaceMode() { return "gateway-conformance-infra" diff --git a/test/e2e/tests/weighted_backend.go b/test/e2e/tests/weighted_backend.go index d20e286e67d..afb46cb6882 100644 --- a/test/e2e/tests/weighted_backend.go +++ b/test/e2e/tests/weighted_backend.go @@ -162,9 +162,13 @@ func testMixedValidAndInvalid(t *testing.T, suite *suite.ConformanceTestSuite) { }) } - // The backendRef without endpoints is dropped, so all the requests are routed to the valid backend. t.Run("MixedValidAndNoEndpoints", func(t *testing.T) { - runMixedValidAndNoEndpointsScenario(t, suite, gwAddr, "/mixed-valid-and-no-endpoints") + if DropBackendRefsWithoutEndpoints() { + // The backendRef without endpoints is dropped, so all the requests are routed to the valid backend. + runMixedValidAndNoEndpointsScenario(t, suite, gwAddr, "/mixed-valid-and-no-endpoints") + } else { + runMixedValidAndInvalidScenario(t, suite, gwAddr, "/mixed-valid-and-no-endpoints", 503) + } }) } From c4f081bb247f7fbc8034cd8dae347380180e2962 Mon Sep 17 00:00:00 2001 From: kkk777-7 Date: Fri, 30 Oct 2026 14:58:07 +0900 Subject: [PATCH 6/6] fix: generate testdata Signed-off-by: kkk777-7 --- .../testdata/httproute-with-backend-priority-filter.out.yaml | 3 ++- site/content/en/latest/api/extension_types.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/gatewayapi/testdata/httproute-with-backend-priority-filter.out.yaml b/internal/gatewayapi/testdata/httproute-with-backend-priority-filter.out.yaml index 43f8fa1fe4e..ce74760bb30 100644 --- a/internal/gatewayapi/testdata/httproute-with-backend-priority-filter.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-backend-priority-filter.out.yaml @@ -557,7 +557,8 @@ xdsIR: name: grpcroute/default/grpc-route-1/rule/0/backend/0 protocol: GRPC weight: 1 - - name: grpcroute/default/grpc-route-1/rule/0/backend/1 + - invalid: true + name: grpcroute/default/grpc-route-1/rule/0/backend/1 weight: 1 hostname: '*' isHTTP2: true diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 695606dd767..acb2c8a9543 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -6569,6 +6569,7 @@ _Appears in:_ | `OIDCTokenCookieChunking` | OIDCTokenCookieChunking splits an OIDC access, ID or refresh token too large for one
browser cookie across several cookies, instead of the browser dropping it and the login
looping back to the OIDC provider. It is enabled by default. Envoy Proxy older than v1.40
can't read the split cookies, so a client that holds such a token in one cookie today, such
as a non-browser client, is sent back to the OIDC provider when it reaches an older proxy.
Disable this flag while proxies of an earlier Envoy Gateway release serve the same
hostnames, for example during an upgrade.
| | `DropBackendRefsWithoutEndpoints` | DropBackendRefsWithoutEndpoints excludes backendRefs without ready endpoints from an HTTPRoute
or GRPCRoute rule when at least one other backendRef in the same rule is available, so the
remaining backends receive all the traffic per their relative weights instead of returning 503
for the share of the excluded backendRefs.
Rules that also have invalid backendRefs are unchanged.
Disabled by default.
| + #### RuntimeFlags