Add a proxy config key for the upstream connection idle timeout

agent/xds/clusters.go

@@ -762,6 +762,8 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
 
 		svcConfig, ok := cfgSnap.TerminatingGateway.ServiceConfigs[svc]
 		isHTTP2 := false
+		var protocol string
+		var idleTimeoutMs *int
 		if ok {
 			upstreamCfg, err := structs.ParseUpstreamConfig(svcConfig.ProxyConfig)
 			if err != nil {
@@ -770,6 +772,15 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
 				s.Logger.Warn("failed to parse", "upstream", svc, "error", err)
 			}
 			isHTTP2 = upstreamCfg.Protocol == "http2" || upstreamCfg.Protocol == "grpc"
+			protocol = upstreamCfg.Protocol
+
+			// A terminating gateway gets proxy-defaults through each linked
+			// service's resolved config, not its own.
+			proxyCfg, err := config.ParseProxyConfig(svcConfig.ProxyConfig)
+			if err != nil {
+				s.Logger.Warn("failed to parse Proxy.Config", "service", svc, "error", err)
+			}
+			idleTimeoutMs = proxyCfg.UpstreamConnectionIdleTimeoutMs
 		}
 
 		var handled bool
@@ -796,6 +807,9 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
 					return nil, err
 				}
 			}
+			if err := s.setUpstreamConnectionIdleTimeout(cluster, protocol, idleTimeoutMs); err != nil {
+				return nil, err
+			}
 			clusters = append(clusters, cluster)
 		}
 
@@ -824,6 +838,9 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
 					return nil, err
 				}
 			}
+			if err := s.setUpstreamConnectionIdleTimeout(cluster, protocol, idleTimeoutMs); err != nil {
+				return nil, err
+			}
 			clusters = append(clusters, cluster)
 		}
 	}
@@ -1365,6 +1382,9 @@ func (s *ResourceGenerator) makeUpstreamClusterForPeerService(
 				return c, err
 			}
 		}
+		if err := s.setUpstreamConnectionIdleTimeout(c, upstreamConfig.Protocol, cfgSnap.GetProxyConfig(s.Logger).UpstreamConnectionIdleTimeoutMs); err != nil {
+			return c, err
+		}
 
 		useEDS := true
 		if _, ok := cfgSnap.ConnectProxy.PeerUpstreamEndpointsUseHostnames[uid]; ok {
@@ -1485,6 +1505,9 @@ func (s *ResourceGenerator) makeUpstreamClusterForPreparedQuery(upstream structs
 				return c, err
 			}
 		}
+		if err := s.setUpstreamConnectionIdleTimeout(c, cfg.Protocol, cfgSnap.GetProxyConfig(s.Logger).UpstreamConnectionIdleTimeoutMs); err != nil {
+			return c, err
+		}
 	}
 
 	endpoints := cfgSnap.ConnectProxy.PreparedQueryEndpoints[uid]
@@ -1788,6 +1811,12 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain(
 					return nil, err
 				}
 			}
+			// Mesh gateways pass connections through without pooling them.
+			if !forMeshGateway {
+				if err := s.setUpstreamConnectionIdleTimeout(c, upstreamConfig.Protocol, cfgSnap.GetProxyConfig(s.Logger).UpstreamConnectionIdleTimeoutMs); err != nil {
+					return nil, err
+				}
+			}
 
 			switch len(groupedTarget.Targets) {
 			case 0:
@@ -2516,6 +2545,47 @@ func (s *ResourceGenerator) setHttp2ProtocolOptions(c *envoy_cluster_v3.Cluster)
 	return nil
 }
 
+// setUpstreamConnectionIdleTimeout applies upstream_connection_idle_timeout_ms
+// to an http, http2 or grpc upstream cluster. It runs after
+// setHttp2ProtocolOptions and keeps the HTTP/2 config that put there. http
+// clusters have no protocol options yet, so they get HTTP/1.1, which is what
+// Envoy uses without them.
+func (s *ResourceGenerator) setUpstreamConnectionIdleTimeout(c *envoy_cluster_v3.Cluster, protocol string, timeoutMs *int) error {
+	if timeoutMs == nil || !structs.IsProtocolHTTPLike(protocol) {
+		return nil
+	}
+
+	const name = "envoy.extensions.upstreams.http.v3.HttpProtocolOptions"
+	cfg := &envoy_upstreams_v3.HttpProtocolOptions{}
+	if existing, ok := c.TypedExtensionProtocolOptions[name]; ok {
+		if err := existing.UnmarshalTo(cfg); err != nil {
+			return err
+		}
+	} else {
+		cfg.UpstreamProtocolOptions = &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig_{
+			ExplicitHttpConfig: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig{
+				ProtocolConfig: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{
+					HttpProtocolOptions: &envoy_core_v3.Http1ProtocolOptions{},
+				},
+			},
+		}
+	}
+	cfg.CommonHttpProtocolOptions = &envoy_core_v3.HttpProtocolOptions{
+		IdleTimeout: durationpb.New(time.Duration(*timeoutMs) * time.Millisecond),
+	}
+
+	any, err := anypb.New(cfg)
+	if err != nil {
+		return err
+	}
+	if c.TypedExtensionProtocolOptions == nil {
+		c.TypedExtensionProtocolOptions = make(map[string]*anypb.Any)
+	}
+	c.TypedExtensionProtocolOptions[name] = any
+
+	return nil
+}
+
 // Allows forwarding either HTTP/1.1 or HTTP/2 traffic to the local application.
 // The protocol used depends on the protocol received from the downstream service
 // on the public listener.

agent/xds/config/config.go

@@ -102,6 +102,12 @@ type ProxyConfig struct {
 	// respected (300s)
 	LocalIdleTimeoutMs *int `mapstructure:"local_idle_timeout_ms"`
 
+	// UpstreamConnectionIdleTimeoutMs is the number of milliseconds a connection
+	// to an http, http2 or grpc upstream can go without requests before the proxy
+	// closes it. If not set or negative, no value is set, Envoy defaults are
+	// respected (1h). 0 disables the timeout.
+	UpstreamConnectionIdleTimeoutMs *int `mapstructure:"upstream_connection_idle_timeout_ms"`
+
 	// Protocol describes the service's protocol. Valid values are "tcp",
 	// "http" and "grpc". Anything else is treated as tcp. This enables
 	// protocol aware features like per-request metrics and connection
@@ -169,6 +175,10 @@ func ParseProxyConfig(m map[string]interface{}) (ProxyConfig, error) {
 	if cfg.LocalConnectTimeoutMs < 1 {
 		cfg.LocalConnectTimeoutMs = 5000
 	}
+	// Envoy rejects a negative duration, and with it every HTTP upstream cluster.
+	if cfg.UpstreamConnectionIdleTimeoutMs != nil && *cfg.UpstreamConnectionIdleTimeoutMs < 0 {
+		cfg.UpstreamConnectionIdleTimeoutMs = nil
+	}
 
 	return cfg, nil
 }

agent/xds/config/config_test.go

@@ -196,6 +196,38 @@ func TestParseProxyConfig(t *testing.T) {
 				Protocol:              "tcp",
 			},
 		},
+		{
+			name: "upstream connection idle timeout override",
+			input: map[string]interface{}{
+				"upstream_connection_idle_timeout_ms": float64(300000),
+			},
+			want: ProxyConfig{
+				LocalConnectTimeoutMs:           5000,
+				UpstreamConnectionIdleTimeoutMs: intPointer(300000),
+				Protocol:                        "tcp",
+			},
+		},
+		{
+			name: "upstream connection idle timeout zero disables it",
+			input: map[string]interface{}{
+				"upstream_connection_idle_timeout_ms": 0,
+			},
+			want: ProxyConfig{
+				LocalConnectTimeoutMs:           5000,
+				UpstreamConnectionIdleTimeoutMs: intPointer(0),
+				Protocol:                        "tcp",
+			},
+		},
+		{
+			name: "upstream connection idle timeout negative is ignored",
+			input: map[string]interface{}{
+				"upstream_connection_idle_timeout_ms": -1,
+			},
+			want: ProxyConfig{
+				LocalConnectTimeoutMs: 5000,
+				Protocol:              "tcp",
+			},
+		},
 		{
 			name: "balance inbound connections override, string",
 			input: map[string]interface{}{

agent/xds/resources_test.go

@@ -2155,6 +2155,28 @@ func getCustomConfigurationGoldenTestCases(enterprise bool) []goldenTestCase {
 				}, nil)
 			},
 		},
+		{
+			// db is http and gets HTTP/1.1 set explicitly; the geo-cache prepared
+			// query stays tcp and is left alone.
+			name: "custom-upstream-connection-idle-timeout",
+			create: func(t testinf.T) *proxycfg.ConfigSnapshot {
+				return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) {
+					ns.Proxy.Config["upstream_connection_idle_timeout_ms"] = 300000
+					ns.Proxy.Upstreams[0].Config["protocol"] = "http"
+				}, nil)
+			},
+		},
+		{
+			// Both keep their HTTP/2 config alongside the idle timeout.
+			name: "custom-upstream-connection-idle-timeout-http2",
+			create: func(t testinf.T) *proxycfg.ConfigSnapshot {
+				return proxycfg.TestConfigSnapshot(t, func(ns *structs.NodeService) {
+					ns.Proxy.Config["upstream_connection_idle_timeout_ms"] = 300000
+					ns.Proxy.Upstreams[0].Config["protocol"] = "http2"
+					ns.Proxy.Upstreams[1].Config = map[string]interface{}{"protocol": "grpc"}
+				}, nil)
+			},
+		},
 		{
 			name: "custom-passive-healthcheck",
 			create: func(t testinf.T) *proxycfg.ConfigSnapshot {
@@ -2476,6 +2498,16 @@ func getTerminatingGatewayPeeringGoldenTestCases() []goldenTestCase {
 			name:   "terminating-gateway-http2-upstream-subsets",
 			create: proxycfg.TestConfigSnapshotTerminatingGatewaySubsetsHTTP2,
 		},
+		{
+			// Covers both the service cluster and its subset clusters.
+			name: "terminating-gateway-upstream-connection-idle-timeout",
+			create: func(t testinf.T) *proxycfg.ConfigSnapshot {
+				snap := proxycfg.TestConfigSnapshotTerminatingGatewaySubsetsHTTP2(t)
+				web := structs.NewServiceName("web", nil)
+				snap.TerminatingGateway.ServiceConfigs[web].ProxyConfig["upstream_connection_idle_timeout_ms"] = 300000
+				return snap
+			},
+		},
 		{
 			name:   "terminating-gateway-ignore-extra-resolvers",
 			create: proxycfg.TestConfigSnapshotTerminatingGatewayIgnoreExtraResolvers,