diff --git a/api/v1alpha1/envoygateway_helpers.go b/api/v1alpha1/envoygateway_helpers.go index 51f0689f40e..2653f2c7874 100644 --- a/api/v1alpha1/envoygateway_helpers.go +++ b/api/v1alpha1/envoygateway_helpers.go @@ -179,10 +179,11 @@ func (r *RateLimit) GetServiceSettings() *ClusterSettings { // defaultRuntimeFlags are the default runtime flags for Envoy Gateway. var defaultRuntimeFlags = map[RuntimeFlag]bool{ - XDSNameSchemeV2: false, - EndpointSliceIndex: true, - EnvoyProxyPatch: true, - OIDCTokenCookieChunking: true, + XDSNameSchemeV2: false, + EndpointSliceIndex: true, + EnvoyProxyPatch: true, + OIDCTokenCookieChunking: true, + DropBackendRefsWithoutEndpoints: false, } // IsEnabled checks if an experimental Gateway API is enabled in the EnvoyGateway configuration. diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index e583e692953..e3c89e647fb 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -170,7 +170,7 @@ type GatewayAPISettings struct { // RuntimeFlag defines a runtime flag used to guard breaking changes or risky experimental features in new Envoy Gateway releases. // A runtime flag may be enabled or disabled by default and can be toggled through the EnvoyGateway resource. // +enum -// +kubebuilder:validation:Enum=XDSNameSchemeV2;EndpointSliceIndex;PerResourceSystemCASecret;EnvoyProxyPatch;ApplyClientTrafficPolicyHTTP1SettingsToClientsOnly;OIDCTokenCookieChunking +// +kubebuilder:validation:Enum=XDSNameSchemeV2;EndpointSliceIndex;PerResourceSystemCASecret;EnvoyProxyPatch;ApplyClientTrafficPolicyHTTP1SettingsToClientsOnly;OIDCTokenCookieChunking;DropBackendRefsWithoutEndpoints type RuntimeFlag string const ( @@ -218,6 +218,14 @@ const ( // Disable this flag while proxies of an earlier Envoy Gateway release serve the same // hostnames, for example during an upgrade. OIDCTokenCookieChunking RuntimeFlag = "OIDCTokenCookieChunking" + + // DropBackendRefsWithoutEndpoints excludes backendRefs without ready endpoints from an HTTPRoute + // or GRPCRoute rule when at least one other backendRef in the same rule is available, so the + // remaining backends receive all the traffic per their relative weights instead of returning 503 + // for the share of the excluded backendRefs. + // Rules that also have invalid backendRefs are unchanged. + // Disabled by default. + DropBackendRefsWithoutEndpoints RuntimeFlag = "DropBackendRefsWithoutEndpoints" ) // RuntimeFlags provide a mechanism to guard breaking changes or risky experimental features in new Envoy Gateway releases. diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index 16942ec64fa..8163c7b4353 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "net" + "slices" "sort" "strconv" "strings" @@ -230,6 +231,14 @@ type routeBackendRefDestination struct { backendClusterKey *BackendClusterKey } +// removeNoEndpointsDestinations returns the destinations without the ones that have no ready endpoints. +// Invalid, dynamic resolver and custom backend destinations are kept as they are handled separately. +func removeNoEndpointsDestinations(destinations []routeBackendRefDestination) []routeBackendRefDestination { + return slices.DeleteFunc(destinations, func(d routeBackendRefDestination) bool { + return !d.ds.Invalid && !d.ds.IsDynamicResolver && !d.ds.IsCustomBackend && len(d.ds.Endpoints) == 0 + }) +} + // httpRouteWithBackendDestinations pairs one rule-match's ir.HTTPRoute with its rule's not-yet-resolved // routeBackendDestinations, deferring the final Settings/BackendClusterRefs split - and thus whether a // merge-eligible backend actually gets a shared cluster - to routeDestinationForListener, once a @@ -385,6 +394,18 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe routeBackendDestinations = append(routeBackendDestinations, backendDest) } + // When DropBackendRefsWithoutEndpoints is enabled and some backendRefs in the rule are available, drop the + // ones without ready endpoints so that traffic is redistributed to the available backends instead of + // returning 503 for their share. + // The BackendsAvailable condition is still set for the dropped backendRefs. + // Skip this when the rule has invalid backendRefs, as the dropped share would also go to the invalid ones + // and change their configured fraction. + if t.DropBackendRefsWithoutEndpoints && + backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 && backendWeights.Invalid == 0 { + routeBackendDestinations = removeNoEndpointsDestinations(routeBackendDestinations) + backendWeights.NoEndpoints = 0 + } + switch { // return 500 if any filter processing error occurred case processFilterError != nil: @@ -1582,6 +1603,7 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe fmt.Errorf("failed to process route rule %d backendRef %d: %w", ruleIdx, i, err), err.Reason(), )) + backendDest.ds.Invalid = true processDestinationError = err } } @@ -1602,6 +1624,18 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe routeBackendDestinations = append(routeBackendDestinations, backendDest) } + // When DropBackendRefsWithoutEndpoints is enabled and some backendRefs in the rule are available, drop the + // ones without ready endpoints so that traffic is redistributed to the available backends instead of + // returning 503 for their share. + // The BackendsAvailable condition is still set for the dropped backendRefs. + // Skip this when the rule has invalid backendRefs, as the dropped share would also go to the invalid ones + // and change their configured fraction. + if t.DropBackendRefsWithoutEndpoints && + backendWeights.Valid > 0 && backendWeights.NoEndpoints > 0 && backendWeights.Invalid == 0 { + routeBackendDestinations = removeNoEndpointsDestinations(routeBackendDestinations) + backendWeights.NoEndpoints = 0 + } + switch { // return 500 if any filter processing error occurred case processFilterError != nil: diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index 9cd555b5ea3..ed95a49344b 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -334,6 +334,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re MergeGateways: gatewayapi.IsMergeGatewaysEnabled(resources), MergeBackends: gatewayapi.ResolveMergeBackendsConfig(resources), PerResourceSystemCASecret: r.EnvoyGateway.RuntimeFlags.IsEnabled(egv1a1.PerResourceSystemCASecret), + DropBackendRefsWithoutEndpoints: r.EnvoyGateway.RuntimeFlags.IsEnabled(egv1a1.DropBackendRefsWithoutEndpoints), WasmCache: r.wasmCache, RunningOnHost: r.EnvoyGateway.Provider != nil && r.EnvoyGateway.Provider.IsRunningOnHost(), InfraRemotelyManaged: r.EnvoyGateway.Provider != nil && r.EnvoyGateway.Provider.IsInfraManagedRemotely(), diff --git a/internal/gatewayapi/testdata/grpcroute-with-mixed-invalid-and-valid-backend-refs.out.yaml b/internal/gatewayapi/testdata/grpcroute-with-mixed-invalid-and-valid-backend-refs.out.yaml index 408e5064430..7fd84be2d5b 100644 --- a/internal/gatewayapi/testdata/grpcroute-with-mixed-invalid-and-valid-backend-refs.out.yaml +++ b/internal/gatewayapi/testdata/grpcroute-with-mixed-invalid-and-valid-backend-refs.out.yaml @@ -161,7 +161,8 @@ xdsIR: name: grpcroute/default/grpcroute-1/rule/0/backend/0 protocol: GRPC weight: 1 - - name: grpcroute/default/grpcroute-1/rule/0/backend/1 + - invalid: true + name: grpcroute/default/grpcroute-1/rule/0/backend/1 weight: 1 headerMatches: - distinct: false diff --git a/internal/gatewayapi/testdata/httproute-with-backend-priority-filter.out.yaml b/internal/gatewayapi/testdata/httproute-with-backend-priority-filter.out.yaml index 43f8fa1fe4e..ce74760bb30 100644 --- a/internal/gatewayapi/testdata/httproute-with-backend-priority-filter.out.yaml +++ b/internal/gatewayapi/testdata/httproute-with-backend-priority-filter.out.yaml @@ -557,7 +557,8 @@ xdsIR: name: grpcroute/default/grpc-route-1/rule/0/backend/0 protocol: GRPC weight: 1 - - name: grpcroute/default/grpc-route-1/rule/0/backend/1 + - invalid: true + name: grpcroute/default/grpc-route-1/rule/0/backend/1 weight: 1 hostname: '*' isHTTP2: true diff --git a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml new file mode 100644 index 00000000000..ae51fada900 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.in.yaml @@ -0,0 +1,105 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + # the backendRef without endpoints is dropped, service-1 receives all the traffic + - matches: + - path: + type: PathPrefix + value: "/some-without-endpoints" + backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + # the backendRef without endpoints is kept to preserve the invalid fraction, the invalid and no endpoints share returns 503 + - matches: + - path: + type: PathPrefix + value: "/mixed-invalid-and-without-endpoints" + backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + - name: service-not-exist + port: 8080 + # all backendRefs have no endpoints, 503 direct response is returned + - matches: + - path: + type: PathPrefix + value: "/all-without-endpoints" + backendRefs: + - name: service-with-no-endpoints + port: 8080 + - name: service-with-no-endpoints + port: 8080 +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: GRPCRoute + metadata: + namespace: default + name: grpcroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + rules: + # the backendRef without endpoints is dropped, service-1 receives all the traffic + - matches: + - method: + service: com.example + method: Example + backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 +services: +- apiVersion: v1 + kind: Service + metadata: + namespace: default + name: service-with-no-endpoints + spec: + clusterIP: 1.1.1.1 + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 +endpointSlices: +- apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + namespace: default + name: endpointslice-service-with-no-endpoints + labels: + kubernetes.io/service-name: service-with-no-endpoints + addressType: IPv4 + ports: + - name: http + port: 8080 + protocol: TCP diff --git a/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml new file mode 100644 index 00000000000..4e22493d511 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-with-multiple-backendrefs-some-without-endpoints.out.yaml @@ -0,0 +1,327 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 2 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: GRPCRoute + metadata: + name: grpcroute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + rules: + - backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + matches: + - method: + method: Example + service: com.example + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: 'Failed to find endpoints: no ready endpoints for the related Service + default/service-with-no-endpoints.' + reason: EndpointsNotFound + status: "False" + type: BackendsAvailable + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: httproute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + rules: + - backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + matches: + - path: + type: PathPrefix + value: /some-without-endpoints + - backendRefs: + - name: service-1 + port: 8080 + - name: service-with-no-endpoints + port: 8080 + - name: service-not-exist + port: 8080 + matches: + - path: + type: PathPrefix + value: /mixed-invalid-and-without-endpoints + - backendRefs: + - name: service-with-no-endpoints + port: 8080 + - name: service-with-no-endpoints + port: 8080 + matches: + - path: + type: PathPrefix + value: /all-without-endpoints + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: 'Failed to process route rule 1 backendRef 2: service default/service-not-exist + not found.' + reason: BackendNotFound + status: "False" + type: ResolvedRefs + - lastTransitionTime: null + message: |- + Failed to find endpoints: no ready endpoints for the related Service default/service-with-no-endpoints. + Failed to find endpoints: no ready endpoints for the related Service default/service-with-no-endpoints. + Failed to find endpoints: no ready endpoints for the related Service default/service-with-no-endpoints. + Failed to find endpoints: no ready endpoints for the related Service default/service-with-no-endpoints. + reason: EndpointsNotFound + status: "False" + type: BackendsAvailable + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + ownerReference: + kind: GatewayClass + name: envoy-gateway-class + name: envoy-gateway/gateway-1 + namespace: envoy-gateway-system +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + globalResources: + proxyServiceCluster: + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + settings: + - addressType: IP + endpoints: + - host: 7.6.5.4 + port: 8080 + zone: zone1 + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + namespace: envoy-gateway-system + sectionName: "8080" + name: envoy-gateway/gateway-1 + protocol: TCP + http: + - address: 0.0.0.0 + externalPort: 80 + grpc: + enableGRPCStats: true + enableGRPCWeb: true + hostnames: + - '*' + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: "8080" + name: grpcroute/default/grpcroute-1/rule/0/backend/0 + protocol: GRPC + weight: 1 + hostname: '*' + isHTTP2: true + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0/match/0/* + pathMatch: + distinct: false + exact: /com.example/Example + name: "" + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: "8080" + name: httproute/default/httproute-1/rule/1/backend/0 + protocol: HTTP + weight: 1 + - name: httproute/default/httproute-1/rule/1/backend/1 + weight: 1 + - invalid: true + name: httproute/default/httproute-1/rule/1/backend/2 + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/match/0/* + pathMatch: + distinct: false + name: "" + prefix: /mixed-invalid-and-without-endpoints + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: "8080" + name: httproute/default/httproute-1/rule/0/backend/0 + protocol: HTTP + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/* + pathMatch: + distinct: false + name: "" + prefix: /some-without-endpoints + - directResponse: + statusCode: 503 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/2/match/0/* + pathMatch: + distinct: false + name: "" + prefix: /all-without-endpoints + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index c677e863c75..91b5988bf46 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -146,6 +146,10 @@ type Translator struct { // sharing a single system_ca_certificates secret. Disabled by default (shared secret used). PerResourceSystemCASecret bool + // DropBackendRefsWithoutEndpoints excludes backendRefs without ready endpoints from a route rule + // when at least one other backendRef in the same rule is available. Disabled by default. + DropBackendRefsWithoutEndpoints bool + // GatewayNamespaceMode is true if controller uses gateway namespace mode for infra deployments. GatewayNamespaceMode bool diff --git a/internal/gatewayapi/translator_test.go b/internal/gatewayapi/translator_test.go index 6b896629d2a..c723bbfc279 100644 --- a/internal/gatewayapi/translator_test.go +++ b/internal/gatewayapi/translator_test.go @@ -61,6 +61,7 @@ func TestTranslate(t *testing.T) { PerResourceSystemCASecret bool RateLimitClusterSettings *egv1a1.ClusterSettings EnvoyProxyPatchDisabled bool + DropBackendRefsWithoutEndpoints bool CTPHTTP1ClientScopeOnly bool OIDCTokenCookieChunkingDisabled bool }{ @@ -177,6 +178,10 @@ func TestTranslate(t *testing.T) { name: "envoyproxy-patch-disabled", EnvoyProxyPatchDisabled: true, }, + { + name: "httproute-with-multiple-backendrefs-some-without-endpoints", + DropBackendRefsWithoutEndpoints: true, + }, { name: "mergebackends-http-merge-listenerset-http1-ctp-client-scope", CTPHTTP1ClientScopeOnly: true, @@ -229,6 +234,7 @@ func TestTranslate(t *testing.T) { perResourceSystemCASecret := false var rateLimitClusterSettings *egv1a1.ClusterSettings envoyProxyPatchDisabled := false + dropBackendRefsWithoutEndpoints := false ctpHTTP1ClientScopeOnly := false oidcTokenCookieChunkingDisabled := false @@ -243,6 +249,7 @@ func TestTranslate(t *testing.T) { perResourceSystemCASecret = config.PerResourceSystemCASecret rateLimitClusterSettings = config.RateLimitClusterSettings envoyProxyPatchDisabled = config.EnvoyProxyPatchDisabled + dropBackendRefsWithoutEndpoints = config.DropBackendRefsWithoutEndpoints ctpHTTP1ClientScopeOnly = config.CTPHTTP1ClientScopeOnly oidcTokenCookieChunkingDisabled = config.OIDCTokenCookieChunkingDisabled } @@ -264,6 +271,7 @@ func TestTranslate(t *testing.T) { ExtensionBackendsEnabled: true, SDSSecretRefEnabled: sdsEnabled, PerResourceSystemCASecret: perResourceSystemCASecret, + DropBackendRefsWithoutEndpoints: dropBackendRefsWithoutEndpoints, ControllerNamespace: "envoy-gateway-system", MergeGateways: IsMergeGatewaysEnabled(resources), MergeBackends: ResolveMergeBackendsConfig(resources), diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index 1917073e679..da333cb829d 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -347,7 +347,8 @@ func buildXdsRouteAction(route *ir.HTTPRoute, backendIndex backendClusterIndex) // Weighted routing is needed whenever there's more than one cluster to route between. if route.NeedsClusterPerSetting() || clusterCount > 1 { - return buildXdsWeightedRouteAction(route.Destination, backendIndex) + // IsHTTP2 is only set for GRPCRoute. + return buildXdsWeightedRouteAction(route.Destination, backendIndex, route.IsHTTP2) } // Defaults to the destination's own name; a single backend cluster overrides it. @@ -362,7 +363,7 @@ func buildXdsRouteAction(route *ir.HTTPRoute, backendIndex backendClusterIndex) } } -func buildXdsWeightedRouteAction(destination *ir.RouteDestination, backendIndex backendClusterIndex) *routev3.RouteAction { +func buildXdsWeightedRouteAction(destination *ir.RouteDestination, backendIndex backendClusterIndex, isGRPC bool) *routev3.RouteAction { weightedClusters := make([]*routev3.WeightedCluster_ClusterWeight, 0) backendWeights := destination.ToBackendWeights() @@ -408,8 +409,10 @@ func buildXdsWeightedRouteAction(destination *ir.RouteDestination, backendIndex // 503 status code should be returned for Services without ready endpoints // Reference: https://gateway-api.sigs.k8s.io/reference/api-spec/main/spec/#httprouterule clusterNotFoundResponseCode := routev3.RouteAction_INTERNAL_SERVER_ERROR - // Envoy can't handle mixed 500 and 503 responses, so we use 503 when both invalid and empty - if backendWeights.NoEndpoints > 0 { + // Envoy can't handle mixed 500 and 503 responses, so we use 503 when both invalid and empty. + // For GRPCRoute, invalid GRPCBackendRefs must receive an UNAVAILABLE status, which Envoy maps from 503. + // Reference: https://gateway-api.sigs.k8s.io/reference/api-spec/main/spec/#grpcrouterule + if backendWeights.NoEndpoints > 0 || isGRPC { clusterNotFoundResponseCode = routev3.RouteAction_SERVICE_UNAVAILABLE } return &routev3.RouteAction{ diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.yaml new file mode 100644 index 00000000000..4a6e3b07ff0 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.yaml @@ -0,0 +1,125 @@ +http: +- address: 0.0.0.0 + externalPort: 80 + grpc: + enableGRPCStats: true + enableGRPCWeb: true + hostnames: + - '*' + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: '8080' + name: grpcroute/default/grpcroute-1/rule/0/backend/0 + protocol: GRPC + weight: 1 + hostname: '*' + isHTTP2: true + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0/match/0/* + pathMatch: + distinct: false + exact: /com.example/Example + name: '' + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: '8080' + name: httproute/default/httproute-1/rule/1/backend/0 + protocol: HTTP + weight: 1 + - invalid: true + name: httproute/default/httproute-1/rule/1/backend/2 + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/match/0/* + pathMatch: + distinct: false + name: '' + prefix: /mixed-invalid-and-without-endpoints + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + kind: Service + name: service-1 + namespace: default + sectionName: '8080' + name: httproute/default/httproute-1/rule/0/backend/0 + protocol: HTTP + weight: 1 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/* + pathMatch: + distinct: false + name: '' + prefix: /some-without-endpoints + - directResponse: + statusCode: 503 + hostname: '*' + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/2/match/0/* + pathMatch: + distinct: false + name: '' + prefix: /all-without-endpoints diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-invalid-backend.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-invalid-backend.yaml index be1a5a6223f..8acccf69909 100644 --- a/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-invalid-backend.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-weighted-invalid-backend.yaml @@ -48,3 +48,17 @@ http: invalid: true - weight: 1 name: "third-route-dest/backend/2" # clusterNotFoundResponseCode should be 503 for mixed empty and invalid cluster + - name: "fourth-route" + hostname: "*" + isHTTP2: true + destination: + name: "fourth-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + weight: 1 + name: "fourth-route-dest/backend/0" + - weight: 1 + name: "fourth-route-dest/backend/1" + invalid: true # clusterNotFoundResponseCode should be 503 for invalid cluster of GRPCRoute diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.clusters.yaml new file mode 100644 index 00000000000..721ede94b10 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.clusters.yaml @@ -0,0 +1,127 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: grpcroute/default/grpcroute-1/rule/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0 + perConnectionBufferLimitBytes: 32768 + type: EDS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: httproute/default/httproute-1/rule/1/backend/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/backend/0 + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: httproute/default/httproute-1/rule/1/backend/2 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/backend/2 + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: httproute/default/httproute-1/rule/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0 + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.endpoints.yaml new file mode 100644 index 00000000000..69521a04791 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.endpoints.yaml @@ -0,0 +1,65 @@ +- clusterName: grpcroute/default/grpcroute-1/rule/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 7.7.7.7 + portValue: 8080 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: grpcroute/default/grpcroute-1/rule/0/backend/0 + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: Service + name: service-1 + namespace: default + sectionName: "8080" +- clusterName: httproute/default/httproute-1/rule/1/backend/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 7.7.7.7 + portValue: 8080 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: httproute/default/httproute-1/rule/1/backend/0 + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: Service + name: service-1 + namespace: default + sectionName: "8080" +- clusterName: httproute/default/httproute-1/rule/1/backend/2 + endpoints: + - loadBalancingWeight: 1 + locality: + region: httproute/default/httproute-1/rule/1/backend/2 +- clusterName: httproute/default/httproute-1/rule/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 7.7.7.7 + portValue: 8080 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: httproute/default/httproute-1/rule/0/backend/0 + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: Service + name: service-1 + namespace: default + sectionName: "8080" diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.listeners.yaml new file mode 100644 index 00000000000..1bf3c32ffbb --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.listeners.yaml @@ -0,0 +1,43 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.grpc_web + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb + - name: envoy.filters.http.grpc_stats + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.grpc_stats.v3.FilterConfig + emitFilterState: true + statsForAllMethods: true + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: envoy-gateway/gateway-1/http + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: envoy-gateway/gateway-1/http + maxConnectionsToAcceptPerSocketEvent: 1 + name: envoy-gateway/gateway-1/http + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.routes.yaml new file mode 100644 index 00000000000..9a1da48ef54 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-multiple-backendrefs-some-without-endpoints.routes.yaml @@ -0,0 +1,73 @@ +- ignorePortInHostMatching: true + name: envoy-gateway/gateway-1/http + virtualHosts: + - domains: + - '*' + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http/* + routes: + - match: + path: /com.example/Example + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0/match/0/* + route: + cluster: grpcroute/default/grpcroute-1/rule/0 + - match: + pathSeparatedPrefix: /mixed-invalid-and-without-endpoints + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/1/match/0/* + route: + clusterNotFoundResponseCode: INTERNAL_SERVER_ERROR + upgradeConfigs: + - upgradeType: websocket + weightedClusters: + clusters: + - name: invalid-backend-cluster + weight: 1 + - name: httproute/default/httproute-1/rule/1/backend/0 + weight: 1 + - match: + pathSeparatedPrefix: /some-without-endpoints + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/* + route: + cluster: httproute/default/httproute-1/rule/0 + upgradeConfigs: + - upgradeType: websocket + - directResponse: + status: 503 + match: + pathSeparatedPrefix: /all-without-endpoints + metadata: + filterMetadata: + envoy-gateway: + resources: + - kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/2/match/0/* diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml index aa81c7ec74a..17415403def 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.clusters.yaml @@ -159,3 +159,49 @@ name: third-route-dest/backend/2 perConnectionBufferLimitBytes: 32768 type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: fourth-route-dest/backend/0 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: fourth-route-dest/backend/0 + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: fourth-route-dest/backend/1 + ignoreHealthOnHostRemoval: true + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: fourth-route-dest/backend/1 + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml index 39278f337cc..80c139f19a1 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.endpoints.yaml @@ -54,3 +54,20 @@ - loadBalancingWeight: 1 locality: region: third-route-dest/backend/2 +- clusterName: fourth-route-dest/backend/0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: fourth-route-dest/backend/0 +- clusterName: fourth-route-dest/backend/1 + endpoints: + - loadBalancingWeight: 1 + locality: + region: fourth-route-dest/backend/1 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml index 10b361451ab..e37e2270dcd 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-weighted-invalid-backend.routes.yaml @@ -42,3 +42,13 @@ weight: 2 - name: third-route-dest/backend/0 weight: 1 + - match: + prefix: / + name: fourth-route + route: + weightedClusters: + clusters: + - name: invalid-backend-cluster + weight: 1 + - name: fourth-route-dest/backend/0 + weight: 1 diff --git a/release-notes/current/new_features/10180-drop-backendrefs-without-ready-endpoints.md b/release-notes/current/new_features/10180-drop-backendrefs-without-ready-endpoints.md new file mode 100644 index 00000000000..27a6e96c5af --- /dev/null +++ b/release-notes/current/new_features/10180-drop-backendrefs-without-ready-endpoints.md @@ -0,0 +1 @@ +Added the `DropBackendRefsWithoutEndpoints` runtime flag (disabled by default). When enabled, HTTPRoute and GRPCRoute rules with multiple backendRefs no longer return 503 for the share of traffic routed to backendRefs without ready endpoints when at least one other backendRef in the same rule is available; those backendRefs are excluded from the rule so the remaining backends receive all the traffic per their relative weights. The `BackendsAvailable=False` route condition is still reported, and a 503 is still returned when all backendRefs in the rule have no ready endpoints. Rules that also have invalid backendRefs are unchanged, so the configured share of the invalid backendRefs is preserved. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index bbd83d7bbd1..acb2c8a9543 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -6567,6 +6567,7 @@ _Appears in:_ | `EnvoyProxyPatch` | EnvoyProxyPatch enables applying the Kubernetes resource `patch` fields configured on
EnvoyProxy's Kubernetes provider settings. It is enabled by default to preserve
pre-existing behavior. Because EnvoyProxy is commonly namespace-scoped and
tenant-authored, a patch may grant arbitrary access to resources applied by Envoy
Gateway's more privileged ServiceAccount; disable this flag in multi-tenant clusters
where tenants can author their own EnvoyProxy resources.
| | `ApplyClientTrafficPolicyHTTP1SettingsToClientsOnly` | ApplyClientTrafficPolicyHTTP1SettingsToClientsOnly restricts the ClientTrafficPolicy http1 fields
(enableTrailers, preserveHeaderCase, http10) to client connections only.
When enabled, these fields no longer affect backend HTTP/1 protocol settings.
Use BackendTrafficPolicy.http1 to configure backend HTTP/1 protocol settings.
Disabled by default to preserve existing behavior. Users are encouraged to migrate
to BackendTrafficPolicy.http1 for backend settings and enable this flag to opt in
to the new behavior. This flag will be enabled by default in a future release.
| | `OIDCTokenCookieChunking` | OIDCTokenCookieChunking splits an OIDC access, ID or refresh token too large for one
browser cookie across several cookies, instead of the browser dropping it and the login
looping back to the OIDC provider. It is enabled by default. Envoy Proxy older than v1.40
can't read the split cookies, so a client that holds such a token in one cookie today, such
as a non-browser client, is sent back to the OIDC provider when it reaches an older proxy.
Disable this flag while proxies of an earlier Envoy Gateway release serve the same
hostnames, for example during an upgrade.
| +| `DropBackendRefsWithoutEndpoints` | DropBackendRefsWithoutEndpoints excludes backendRefs without ready endpoints from an HTTPRoute
or GRPCRoute rule when at least one other backendRef in the same rule is available, so the
remaining backends receive all the traffic per their relative weights instead of returning 503
for the share of the excluded backendRefs.
Rules that also have invalid backendRefs are unchanged.
Disabled by default.
| #### RuntimeFlags diff --git a/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml b/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml index f7048a6571f..55a2f4f5cb1 100644 --- a/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml +++ b/test/config/envoy-gateaway-config/xds-name-scheme-v2.yaml @@ -35,6 +35,7 @@ data: enabled: - XDSNameSchemeV2 - EnvoyProxyPatch + - DropBackendRefsWithoutEndpoints extensionManager: maxMessageSize: 1000M policyResources: diff --git a/test/config/helm/xds-name-scheme-v2.yaml b/test/config/helm/xds-name-scheme-v2.yaml index 290d342c3e9..62f8c2101d4 100644 --- a/test/config/helm/xds-name-scheme-v2.yaml +++ b/test/config/helm/xds-name-scheme-v2.yaml @@ -3,3 +3,4 @@ config: runtimeFlags: enabled: - XDSNameSchemeV2 + - DropBackendRefsWithoutEndpoints diff --git a/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml b/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml index c02adeea395..7190e5ade78 100644 --- a/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml +++ b/test/e2e/testdata/weighted-backend-mixed-valid-and-invalid.yaml @@ -26,7 +26,7 @@ spec: - name: infra-backend-v1 port: 8080 weight: 9 - - name: infra-backend-without-endpoints # should return 503 for empty cluster + - name: infra-backend-without-endpoints # should return 503 for empty cluster, or be dropped when DropBackendRefsWithoutEndpoints is enabled port: 8080 weight: 1 --- diff --git a/test/e2e/tests/utils.go b/test/e2e/tests/utils.go index 6bbb5b04eb1..63092923b99 100644 --- a/test/e2e/tests/utils.go +++ b/test/e2e/tests/utils.go @@ -1276,6 +1276,13 @@ func XDSNameSchemeV2() bool { return DeployProfile == "xds-name-scheme-v2" } +// DropBackendRefsWithoutEndpoints returns true if the DropBackendRefsWithoutEndpoints runtime flag is enabled +// in the deploy profile. +// TODO: remove this after the feature flag is removed. +func DropBackendRefsWithoutEndpoints() bool { + return DeployProfile == "xds-name-scheme-v2" +} + func GetGatewayResourceNamespace() string { if IsGatewayNamespaceMode() { return "gateway-conformance-infra" diff --git a/test/e2e/tests/weighted_backend.go b/test/e2e/tests/weighted_backend.go index 4bf5e59ea69..afb46cb6882 100644 --- a/test/e2e/tests/weighted_backend.go +++ b/test/e2e/tests/weighted_backend.go @@ -154,11 +154,6 @@ func testMixedValidAndInvalid(t *testing.T, suite *suite.ConformanceTestSuite) { path: "/mixed-valid-and-invalid", failureCode: 500, }, - { - name: "MixedValidAndNoEndpoints", - path: "/mixed-valid-and-no-endpoints", - failureCode: 503, - }, } for _, scenario := range scenarios { @@ -166,6 +161,15 @@ func testMixedValidAndInvalid(t *testing.T, suite *suite.ConformanceTestSuite) { runMixedValidAndInvalidScenario(t, suite, gwAddr, scenario.path, scenario.failureCode) }) } + + t.Run("MixedValidAndNoEndpoints", func(t *testing.T) { + if DropBackendRefsWithoutEndpoints() { + // The backendRef without endpoints is dropped, so all the requests are routed to the valid backend. + runMixedValidAndNoEndpointsScenario(t, suite, gwAddr, "/mixed-valid-and-no-endpoints") + } else { + runMixedValidAndInvalidScenario(t, suite, gwAddr, "/mixed-valid-and-no-endpoints", 503) + } + }) } const ( @@ -228,3 +232,32 @@ func runMixedValidAndInvalidScenario(t *testing.T, suite *suite.ConformanceTestS t.Logf("success count for %s is %d, failure count for %s is %d", path, successCount, path, failureCount) } + +func runMixedValidAndNoEndpointsScenario(t *testing.T, suite *suite.ConformanceTestSuite, gwAddr, path string) { + t.Helper() + + expected := http.ExpectedResponse{ + Request: http.Request{ + Path: path, + }, + Response: http.Response{ + StatusCodes: []int{200}, + }, + Namespace: ConformanceInfraNamespace, + } + + // Make sure the valid(response 200) backend are ready. + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expected) + + req := http.MakeRequest(t, &expected, gwAddr, "HTTP", "http") + for i := 0; i < mixedValidAndInvalidRequests; i++ { + _, response, err := suite.RoundTripper.CaptureRoundTrip(req) + if err != nil { + t.Errorf("failed to get expected response: %v", err) + continue + } + if response.StatusCode != 200 { + t.Errorf("unexpected status code %d for %s", response.StatusCode, path) + } + } +}