From 1ca2393f22aa3a4a1575d0ce277b1b3f066fc70f Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Mon, 10 Aug 2026 17:24:32 -0700 Subject: [PATCH 01/12] adding nexus interceptor skeleton --- common/rpc/interceptor/nexus.go | 235 +++++++++++++++++++++++++++ common/rpc/interceptor/nexus_test.go | 61 +++++++ 2 files changed, 296 insertions(+) create mode 100644 common/rpc/interceptor/nexus.go create mode 100644 common/rpc/interceptor/nexus_test.go diff --git a/common/rpc/interceptor/nexus.go b/common/rpc/interceptor/nexus.go new file mode 100644 index 00000000000..c0ddbcf4f13 --- /dev/null +++ b/common/rpc/interceptor/nexus.go @@ -0,0 +1,235 @@ +package interceptor + +import ( + "context" + "errors" + "net/http" + "slices" + + "github.com/nexus-rpc/sdk-go/nexus" + "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/nexus/nexusrpc" +) + +type NexusHandlerFunc func(ctx context.Context, in NexusInterceptorInput) (any, error) + +type NexusInterceptor func(ctx context.Context, in NexusInterceptorInput, next NexusHandlerFunc) (any, error) + +type NexusInterceptorInput interface { + ServiceName() string + OperationName() string + NamespaceName() string + ForwardingInfo() NexusForwardingInfo + sealNexusOp() +} + +var ( + _ NexusInterceptorInput = StartNexusOpInput{} + _ NexusInterceptorInput = CancelNexusOpInput{} + _ NexusInterceptorInput = CompleteNexusOpInput{} +) + +// NexusForwardingInfo contains the request data needed to forward a Nexus operation. +type NexusForwardingInfo struct { + OriginalRequestHeaders http.Header + TaskQueue string + EndpointID string + EndpointName string + BusinessID string +} + +type InterceptorError struct { + // wrapped error + Err error + // Outcome tag for metrics reporting, (draft-review: should Outcomes be enum or at least constants instead) + Outcome string +} + +func (t *InterceptorError) Error() string { + return t.Err.Error() +} + +func (t *InterceptorError) Unwrap() error { + return t.Err +} + +// container for ServiceName(), OperationName(), NamespaceName(), ForwardingInfo() +type nexusOpBase struct { + serviceName, operation, namespaceName string + forwardingInfo NexusForwardingInfo +} + +func (b *nexusOpBase) WithForwardingInfo(info NexusForwardingInfo) { + b.forwardingInfo = info +} + +func (b nexusOpBase) ServiceName() string { + return b.serviceName +} + +func (b nexusOpBase) OperationName() string { + return b.operation +} + +func (b nexusOpBase) NamespaceName() string { + return b.namespaceName +} + +func (b nexusOpBase) ForwardingInfo() NexusForwardingInfo { + return b.forwardingInfo +} + +func (nexusOpBase) sealNexusOp() {} + +func NexusHeaderFromInterceptorInput(in NexusInterceptorInput) (headers.HeaderGetter, error) { + switch opts := in.(type) { + case StartNexusOpInput: + return opts.StartOperationOptions.Header, nil + case CancelNexusOpInput: + return opts.CancelOperationOptions.Header, nil + case CompleteNexusOpInput: + if opts.CompletionRequest == nil || opts.CompletionRequest.HTTPRequest == nil { + return nil, errors.New("Nexus completion request not found") + } + return opts.CompletionRequest.HTTPRequest.Header, nil + default: + return nil, errors.New("unknown Nexus interceptor input") + } +} + +// draft-review: verify that these are the "right" methods/names +func NexusMethodName(in NexusInterceptorInput) string { + switch in.(type) { + case StartNexusOpInput: + return "StartNexusOperation" + case CancelNexusOpInput: + return "CancelNexusOperation" + case CompleteNexusOpInput: + return "CompleteNexusOperation" + default: + return "" + } +} + +type StartNexusOpInput struct { + nexusOpBase + StartOperationOptions nexus.StartOperationOptions + StartOperationInput *nexus.LazyValue +} + +func NewStartNexusOpInput( + serviceName string, + operation string, + namespaceName string, + options nexus.StartOperationOptions, + input *nexus.LazyValue, +) StartNexusOpInput { + return StartNexusOpInput{ + nexusOpBase: nexusOpBase{ + serviceName: serviceName, + operation: operation, + namespaceName: namespaceName, + }, + StartOperationOptions: options, + StartOperationInput: input, + } +} + +type CancelNexusOpInput struct { + nexusOpBase + CancelOperationOptions nexus.CancelOperationOptions + CancellationToken string +} + +func NewCancelNexusOpInput( + serviceName string, + operation string, + namespaceName string, + options nexus.CancelOperationOptions, + cancellationToken string, +) CancelNexusOpInput { + return CancelNexusOpInput{ + nexusOpBase: nexusOpBase{ + serviceName: serviceName, + operation: operation, + namespaceName: namespaceName, + }, + CancelOperationOptions: options, + CancellationToken: cancellationToken, + } +} + +type CompleteNexusOpInput struct { + nexusOpBase + CompletionRequest *nexusrpc.CompletionRequest +} + +// draft-review: Complete doesnt need servicename/op - verify +func NewCompleteNexusOpInput( + namespaceName string, + request *nexusrpc.CompletionRequest, +) CompleteNexusOpInput { + return CompleteNexusOpInput{ + nexusOpBase: nexusOpBase{ + namespaceName: namespaceName, + }, + CompletionRequest: request, + } +} + +func ChainNexusInterceptors(final NexusHandlerFunc, chain []NexusInterceptor) NexusHandlerFunc { + for _, curr := range slices.Backward(chain) { + next := final + final = func(ctx context.Context, opts NexusInterceptorInput) (any, error) { + return curr(ctx, opts, next) + } + } + return final +} + +type nexusAPINameContextKey struct{} +type nexusEndpointNameContextKey struct{} + +// draft-review: only endpoint and apiName are unknowable - the namespace we should be able to get via lookup +type nexusNamespaceContextKey struct{} + +// WithNexusAPIName adds the internal Nexus API name to a request context. +func WithNexusAPIName(ctx context.Context, apiName string) context.Context { + return context.WithValue(ctx, nexusAPINameContextKey{}, apiName) +} + +func NexusAPINameFromContext(ctx context.Context) (string, error) { + apiName, ok := ctx.Value(nexusAPINameContextKey{}).(string) + if !ok { + return "", errors.New("Nexus API name not found in context") + } + return apiName, nil +} + +// WithNexusEndpointName adds the resolved Nexus endpoint name to a request context. +func WithNexusEndpointName(ctx context.Context, endpointName string) context.Context { + return context.WithValue(ctx, nexusEndpointNameContextKey{}, endpointName) +} + +func NexusEndpointNameFromContext(ctx context.Context) (string, error) { + endpointName, ok := ctx.Value(nexusEndpointNameContextKey{}).(string) + if !ok { + return "", errors.New("Nexus endpoint name not found in context") + } + return endpointName, nil +} + +// WithNexusNamespace adds the resolved namespace to a request context. +func WithNexusNamespace(ctx context.Context, namespaceEntry *namespace.Namespace) context.Context { + return context.WithValue(ctx, nexusNamespaceContextKey{}, namespaceEntry) +} + +// darft-review: ideally, there is some utility to lookup by name -> Namespace +func NexusNamespaceFromContext(ctx context.Context) (*namespace.Namespace, error) { + namespaceEntry, ok := ctx.Value(nexusNamespaceContextKey{}).(*namespace.Namespace) + if !ok { + return nil, errors.New("Nexus namespace not found in context") + } + return namespaceEntry, nil +} diff --git a/common/rpc/interceptor/nexus_test.go b/common/rpc/interceptor/nexus_test.go new file mode 100644 index 00000000000..12f7d9526a8 --- /dev/null +++ b/common/rpc/interceptor/nexus_test.go @@ -0,0 +1,61 @@ +package interceptor + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestChainNexusInterceptors(t *testing.T) { + var calls []string + chain := []NexusInterceptor{ + func(ctx context.Context, in NexusInterceptorInput, next NexusHandlerFunc) (any, error) { + calls = append(calls, "first-before") + result, err := next(ctx, in) + calls = append(calls, "first-after") + return result, err + }, + func(ctx context.Context, in NexusInterceptorInput, next NexusHandlerFunc) (any, error) { + calls = append(calls, "second-before") + result, err := next(ctx, in) + calls = append(calls, "second-after") + return result, err + }, + } + + result, err := ChainNexusInterceptors(func(context.Context, NexusInterceptorInput) (any, error) { + calls = append(calls, "handler") + return "result", nil + }, chain)(context.Background(), StartNexusOpInput{}) + + require.NoError(t, err) + require.Equal(t, "result", result) + require.Equal(t, []string{ + "first-before", + "second-before", + "handler", + "second-after", + "first-after", + }, calls) +} + +func TestChainNexusInterceptorsShortCircuit(t *testing.T) { + var calls []string + chain := []NexusInterceptor{ + func(context.Context, NexusInterceptorInput, NexusHandlerFunc) (any, error) { + calls = append(calls, "interceptor") + // dont call next - just return + return "intercepted", nil + }, + } + + result, err := ChainNexusInterceptors(func(context.Context, NexusInterceptorInput) (any, error) { + calls = append(calls, "handler") + return "handler", nil + }, chain)(context.Background(), StartNexusOpInput{}) + + require.NoError(t, err) + require.Equal(t, "intercepted", result) + require.Equal(t, []string{"interceptor"}, calls) +} From bb458800621fd57d042454a1c4fe2ae916ac15a8 Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Mon, 10 Aug 2026 17:31:00 -0700 Subject: [PATCH 02/12] adding nexus handlers to interceptors --- common/authorization/interceptor.go | 98 +++++++++++------ common/rpc/interceptor/caller_info.go | 14 +++ .../interceptor/concurrent_request_limit.go | 24 +++++ common/rpc/interceptor/namespace_logger.go | 6 +- .../rpc/interceptor/namespace_rate_limit.go | 26 +++++ common/rpc/interceptor/namespace_validator.go | 24 +++++ common/rpc/interceptor/rate_limit.go | 24 +++++ common/rpc/interceptor/sdk_version.go | 21 ++++ common/rpc/interceptor/telemetry.go | 101 ++++++++++++++++++ common/rpc/tlsinfo/context.go | 35 ++++++ 10 files changed, 339 insertions(+), 34 deletions(-) create mode 100644 common/rpc/tlsinfo/context.go diff --git a/common/authorization/interceptor.go b/common/authorization/interceptor.go index 48a106f87b5..174314212bf 100644 --- a/common/authorization/interceptor.go +++ b/common/authorization/interceptor.go @@ -3,8 +3,8 @@ package authorization import ( "cmp" "context" - "crypto/x509" "crypto/x509/pkix" + "errors" "time" commonpb "go.temporal.io/api/common/v1" @@ -18,9 +18,11 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/rpc/interceptor" + "go.temporal.io/server/common/rpc/tlsinfo" "google.golang.org/grpc" "google.golang.org/grpc/credentials" - "google.golang.org/grpc/peer" ) type ( @@ -54,32 +56,6 @@ var ( AuthHeader contextKeyAuthHeader ) -// TLSInfoFromContext extracts TLS information from the context's peer value. -func TLSInfoFromContext(ctx context.Context) *credentials.TLSInfo { - p, ok := peer.FromContext(ctx) - if !ok { - return nil - } - if tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo); ok { - return &tlsInfo - } - return nil -} - -// PeerCert extracts an x509 certificate from given tlsInfo. -func PeerCert(tlsInfo *credentials.TLSInfo) *x509.Certificate { - if tlsInfo == nil || len(tlsInfo.State.VerifiedChains) == 0 || len(tlsInfo.State.VerifiedChains[0]) == 0 { - return nil - } - // The assumption here is that we only expect a single verified chain of certs (first[0]). - // It's unclear how we should handle a situation when more than one chain is presented, - // which subject to use. It's okay for us to limit ourselves to one chain. - // We can always extend this logic later. - // We take the first element in the chain ([0]) because that's the client cert - // (at the beginning of the chain), not intermediary CAs or the root CA (at the end of the chain). - return tlsInfo.State.VerifiedChains[0][0] -} - type Interceptor struct { claimMapper ClaimMapper authorizer Authorizer @@ -132,7 +108,7 @@ func (a *Interceptor) Intercept( info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ) (any, error) { - tlsConnection := TLSInfoFromContext(ctx) + tlsConnection := tlsinfo.FromContext(ctx) authInfo := a.GetAuthInfo(tlsConnection, headers.NewGRPCHeaderGetter(ctx), func() string { if a.audienceGetter != nil { @@ -184,6 +160,66 @@ func (a *Interceptor) Intercept( return handler(ctx, req) } +func (a *Interceptor) InterceptNexus( + ctx context.Context, + in interceptor.NexusInterceptorInput, + next interceptor.NexusHandlerFunc, +) (any, error) { + a.logger.Debug("authorizing request") + if a.authorizer == nil { + return next(ctx, in) + } + namespaceName := in.NamespaceName() + apiName, err := interceptor.NexusAPINameFromContext(ctx) + if err != nil { + return nil, &interceptor.InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, false), + Outcome: "internal_auth_error", + } + } + endpointName, err := interceptor.NexusEndpointNameFromContext(ctx) + if err != nil { + return nil, &interceptor.InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, false), + Outcome: "internal_auth_error", + } + } + claims, _ := ctx.Value(MappedClaims).(*Claims) + var req any + // draft-review: check if this might be required to preserve compatibility for custom authorizers + // or if its ok since an interface was not already used instead + // switch in.(type) { + // case interceptor.StartNexusOpInput, interceptor.CancelNexusOpInput: + // case *interceptor.CancelNexusOpInput: + // } + req = in + ct := &CallTarget{ + APIName: apiName, + NexusEndpointName: endpointName, + Namespace: namespaceName, + Request: req, + } + principal, err := a.Authorize(ctx, claims, ct) + if err != nil { + if permissionDeniedError, ok := errors.AsType[*serviceerror.PermissionDenied](err); ok { + a.logger.Debug("Request unauthorized") + return nil, &interceptor.InterceptorError{ + Err: commonnexus.AdaptAuthorizeError(permissionDeniedError), + Outcome: "unauthorized", + } + } + a.logger.Error("Authorization internal error with processing nexus request", tag.Error(err)) + return nil, &interceptor.InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, false), + Outcome: "internal_auth_error", + } + } + if a.enablePrincipalPropagation != nil && a.enablePrincipalPropagation(namespaceName) && principal != nil { + ctx = headers.SetPrincipal(ctx, principal) + } + return next(ctx, in) +} + // InterceptStream is a gRPC stream server interceptor that enforces authorization. func (a *Interceptor) InterceptStream( srv any, @@ -194,7 +230,7 @@ func (a *Interceptor) InterceptStream( ctx := ss.Context() bypassAuth := a.disableStreamingAuthorizer() if !bypassAuth { - tlsConnection := TLSInfoFromContext(ctx) + tlsConnection := tlsinfo.FromContext(ctx) headerGetter := headers.NewGRPCHeaderGetter(ctx) authInfo := a.GetAuthInfo(tlsConnection, headerGetter, func() string { @@ -260,7 +296,7 @@ func (a *Interceptor) GetAuthInfo(tlsConnection *credentials.TLSInfo, header hea authHeader = header.Get(a.authHeaderName) authExtraHeader = header.Get(a.authExtraHeaderName) } - clientCert := PeerCert(tlsConnection) + clientCert := tlsinfo.PeerCert(tlsConnection) if clientCert != nil { tlsSubject = &clientCert.Subject } diff --git a/common/rpc/interceptor/caller_info.go b/common/rpc/interceptor/caller_info.go index a8937f7b4b4..30c0e253aed 100644 --- a/common/rpc/interceptor/caller_info.go +++ b/common/rpc/interceptor/caller_info.go @@ -40,6 +40,20 @@ func (i *CallerInfoInterceptor) Intercept( return handler(ctx, req) } +// InterceptNexus adds caller information for a Nexus request. +func (i *CallerInfoInterceptor) InterceptNexus( + ctx context.Context, + in NexusInterceptorInput, + next NexusHandlerFunc, +) (any, error) { + ctx = PopulateCallerInfo( + ctx, + in.NamespaceName, + func() string { return NexusMethodName(in) }, + ) + return next(headers.Propagate(ctx), in) +} + // PopulateCallerInfo gets current caller info value from the context and updates any that are missing. // Namespace name and method are passed as functions to avoid expensive lookups if those values are already set. func PopulateCallerInfo( diff --git a/common/rpc/interceptor/concurrent_request_limit.go b/common/rpc/interceptor/concurrent_request_limit.go index 0014c794ea6..cd0390e41a8 100644 --- a/common/rpc/interceptor/concurrent_request_limit.go +++ b/common/rpc/interceptor/concurrent_request_limit.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/quotas/calculator" "google.golang.org/grpc" ) @@ -116,6 +117,29 @@ func (ni *ConcurrentRequestLimitInterceptor) Allow( return cleanup, nil } +// InterceptNexus enforces the namespace concurrent-request limit for a Nexus request. +func (ni *ConcurrentRequestLimitInterceptor) InterceptNexus( + ctx context.Context, + in NexusInterceptorInput, + next NexusHandlerFunc, +) (any, error) { + apiName, err := NexusAPINameFromContext(ctx) + if err != nil { + return nil, err + } + metricsHandler := GetMetricsHandlerFromContext(ctx, ni.logger) + // draft-review: this looks safe to pass "in" as any, but confirm in review + cleanup, err := ni.Allow(namespace.Name(in.NamespaceName()), apiName, metricsHandler, in) + defer cleanup() + if err != nil { + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, false), + Outcome: "namespace_concurrency_limited", + } + } + return next(ctx, in) +} + func (ni *ConcurrentRequestLimitInterceptor) counter( namespace namespace.Name, methodName string, diff --git a/common/rpc/interceptor/namespace_logger.go b/common/rpc/interceptor/namespace_logger.go index 8f5a99f1ec8..65b4698839f 100644 --- a/common/rpc/interceptor/namespace_logger.go +++ b/common/rpc/interceptor/namespace_logger.go @@ -6,10 +6,10 @@ import ( "fmt" "go.temporal.io/server/common/api" - "go.temporal.io/server/common/authorization" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/tlsinfo" "google.golang.org/grpc" ) @@ -40,12 +40,12 @@ func (nli *NamespaceLogInterceptor) Intercept( if nli.logger != nil { methodName := api.MethodName(info.FullMethod) namespace := MustGetNamespaceName(nli.namespaceRegistry, req) - tlsInfo := authorization.TLSInfoFromContext(ctx) + tlsInfo := tlsinfo.FromContext(ctx) var serverName string var certThumbprint string if tlsInfo != nil { serverName = tlsInfo.State.ServerName - cert := authorization.PeerCert(tlsInfo) + cert := tlsinfo.PeerCert(tlsInfo) if cert != nil { certThumbprint = fmt.Sprintf("%x", md5.Sum(cert.Raw)) } diff --git a/common/rpc/interceptor/namespace_rate_limit.go b/common/rpc/interceptor/namespace_rate_limit.go index 080e9f667b6..3b86c347bb7 100644 --- a/common/rpc/interceptor/namespace_rate_limit.go +++ b/common/rpc/interceptor/namespace_rate_limit.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/server/common/headers" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/quotas" "go.temporal.io/server/service/frontend/configs" "google.golang.org/grpc" @@ -75,6 +76,8 @@ type ( headerGetter headers.HeaderGetter, numToken int, ) error + + InterceptNexus(ctx context.Context, in NexusInterceptorInput, next NexusHandlerFunc) (resp any, err error) } NamespaceRateLimitInterceptorImpl struct { @@ -225,6 +228,29 @@ func (ni *NamespaceRateLimitInterceptorImpl) AllowN( return nil } +// InterceptNexus enforces the namespace rate limit for a Nexus request. +func (ni *NamespaceRateLimitInterceptorImpl) InterceptNexus( + ctx context.Context, + in NexusInterceptorInput, + next NexusHandlerFunc, +) (any, error) { + apiName, err := NexusAPINameFromContext(ctx) + if err != nil { + return nil, err + } + header, err := NexusHeaderFromInterceptorInput(in) + if err != nil { + return nil, err + } + if err := ni.Allow(namespace.Name(in.NamespaceName()), apiName, header); err != nil { + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "namespace_rate_limited", + } + } + return next(ctx, in) +} + func IsLongPollGetWorkflowExecutionHistoryRequest( req any, ) bool { diff --git a/common/rpc/interceptor/namespace_validator.go b/common/rpc/interceptor/namespace_validator.go index 6928e5e0268..90a29fe7adc 100644 --- a/common/rpc/interceptor/namespace_validator.go +++ b/common/rpc/interceptor/namespace_validator.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/server/common/api" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/namespace" + commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/tasktoken" "google.golang.org/grpc" ) @@ -230,6 +231,29 @@ func (ni *NamespaceValidatorInterceptor) ValidateState(namespaceEntry *namespace return ni.checkReplicationState(namespaceEntry, fullMethod, businessID) } +// InterceptNexus validates the namespace state for a Nexus request. +func (ni *NamespaceValidatorInterceptor) InterceptNexus( + ctx context.Context, + in NexusInterceptorInput, + next NexusHandlerFunc, +) (any, error) { + namespaceEntry, err := NexusNamespaceFromContext(ctx) + if err != nil { + return nil, err + } + apiName, err := NexusAPINameFromContext(ctx) + if err != nil { + return nil, err + } + if err := ni.ValidateState(namespaceEntry, apiName, in.ForwardingInfo().BusinessID); err != nil { + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, false), + Outcome: "invalid_namespace_state", + } + } + return next(ctx, in) +} + func (ni *NamespaceValidatorInterceptor) extractNamespace(req any) (*namespace.Namespace, error) { // Token namespace has priority over request namespace. Check it first. tokenNamespaceEntry, tokenErr := ni.extractNamespaceFromTaskToken(req) diff --git a/common/rpc/interceptor/rate_limit.go b/common/rpc/interceptor/rate_limit.go index 2594c03cb6a..132abd6d620 100644 --- a/common/rpc/interceptor/rate_limit.go +++ b/common/rpc/interceptor/rate_limit.go @@ -8,6 +8,7 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common/headers" + commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/quotas" "google.golang.org/grpc" ) @@ -90,3 +91,26 @@ func (i *RateLimitInterceptor) Allow( } return nil } + +// InterceptNexus enforces the global rate limit for a Nexus request. +func (i *RateLimitInterceptor) InterceptNexus( + ctx context.Context, + in NexusInterceptorInput, + next NexusHandlerFunc, +) (any, error) { + apiName, err := NexusAPINameFromContext(ctx) + if err != nil { + return nil, err + } + header, err := NexusHeaderFromInterceptorInput(in) + if err != nil { + return nil, err + } + if err := i.Allow(apiName, header); err != nil { + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "global_rate_limited", + } + } + return next(ctx, in) +} diff --git a/common/rpc/interceptor/sdk_version.go b/common/rpc/interceptor/sdk_version.go index 6fdb34ffa8f..7106fcc2b3a 100644 --- a/common/rpc/interceptor/sdk_version.go +++ b/common/rpc/interceptor/sdk_version.go @@ -5,6 +5,7 @@ import ( "sync" "go.temporal.io/server/common/headers" + commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/versioninfo" "google.golang.org/grpc" ) @@ -44,6 +45,26 @@ func (vi *SDKVersionInterceptor) Intercept( return handler(ctx, req) } +// InterceptNexus records and validates the SDK version for a Nexus request. +func (vi *SDKVersionInterceptor) InterceptNexus( + ctx context.Context, + in NexusInterceptorInput, + next NexusHandlerFunc, +) (any, error) { + // draft-review: RecordSDKInfo didnt exist before, nice to add + sdkName, sdkVersion := headers.GetClientNameAndVersion(ctx) + if sdkName != "" && sdkVersion != "" { + vi.RecordSDKInfo(sdkName, sdkVersion) + } + if err := vi.versionChecker.ClientSupported(ctx); err != nil { + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "unsupported_client", + } + } + return next(ctx, in) +} + // RecordSDKInfo records name and version tuple in memory func (vi *SDKVersionInterceptor) RecordSDKInfo(name, version string) { info := versioninfo.SDKInfo{Name: name, Version: version} diff --git a/common/rpc/interceptor/telemetry.go b/common/rpc/interceptor/telemetry.go index f906da64032..ae110e9a8b3 100644 --- a/common/rpc/interceptor/telemetry.go +++ b/common/rpc/interceptor/telemetry.go @@ -2,6 +2,9 @@ package interceptor import ( "context" + "errors" + "fmt" + "runtime/debug" "strings" "time" @@ -25,6 +28,8 @@ import ( type ( metricsContextKey struct{} + telemetryContextKey struct{} + TelemetryInterceptor struct { namespaceRegistry namespace.Registry metricsHandler metrics.Handler @@ -33,6 +38,18 @@ type ( logAllReqErrors dynamicconfig.BoolPropertyFnWithNamespaceFilter requestErrorHandler ErrorHandler } + + TelemetryContext interface { + MetricsHandler(error) metrics.Handler + MetricsHandlerForInterceptors() metrics.Handler + MetricsLogger() log.Logger + SetMetricsOutcome(string) + // SetFailureSource records which side produced a failure. Only the start/cancel + // handlers return this to the caller; the completion handler discards it. + SetFailureSource(string) + // HandleRequestError reports a failed request to the shared ErrorHandler. + HandleRequestError(error) + } ) var ( @@ -204,6 +221,90 @@ func AddTelemetryContext(ctx context.Context, metricsHandler metrics.Handler) co return context.WithValue(ctx, metricsCtxKey, metricsHandler) } +// WithTelemetryContext returns a context with telemetry for interceptors that need to +// record their own telemetry - like the forwarder interceptor. +func WithTelemetryContext(ctx context.Context, telemetryContext TelemetryContext) context.Context { + return context.WithValue(ctx, telemetryContextKey{}, telemetryContext) +} + +func TelemetryContextFromContext(ctx context.Context) (TelemetryContext, error) { + telemetryContext, ok := ctx.Value(telemetryContextKey{}).(TelemetryContext) + if !ok { + return nil, errors.New("telemetry context not found") + } + return telemetryContext, nil +} + +// InterceptNexus records request metrics and recovers panics for a Nexus request. +// It runs outermost in the chain, so metrics are recorded in the source cluster even +// when the forwarder redirects. Forwarded requests are distinguished by the +// "request_forwarded" outcome tag rather than by being omitted. +// It also publishes the metrics context that downstream interceptors read via +// GetMetricsHandlerFromContext. +func (ti *TelemetryInterceptor) InterceptNexus( + ctx context.Context, + in NexusInterceptorInput, + next NexusHandlerFunc, +) (out any, retErr error) { + + // draft-review: it it not worth splitting the metrics into pre and post forwarder interceptor groups. + // The only "additional" telemetry from fwder is in the case of an actual redirect happening- + // this should anyway be additional metric and get captured in both original and redirected + // clusters as the request did indeed get handled in both places. + // If there is some reason why this should be avoided, then split them such that + // we just "add" the telemetry context as the outermost and then the recorder section + // is added after the authz and fwder interceptors + + telemetryContext, err := TelemetryContextFromContext(ctx) + if err != nil { + return nil, err + } + // required for forwarder interceptor to grab metrics handle when reporting + ctx = metrics.AddMetricsContext(ctx) + ctx = AddTelemetryContext(ctx, telemetryContext.MetricsHandlerForInterceptors()) + interceptorMetricsHandler := telemetryContext.MetricsHandlerForInterceptors() + metrics.ServiceRequests.With(interceptorMetricsHandler).Record(1) + + startTime := time.Now().UTC() + + defer func() { + reportErr := retErr + if taggedErr, ok := errors.AsType[*InterceptorError](retErr); ok { + telemetryContext.SetMetricsOutcome(taggedErr.Outcome) + reportErr = taggedErr.Err + } + metricsHandler := telemetryContext.MetricsHandler(reportErr) + switch in.(type) { + case CompleteNexusOpInput: + metricsHandler.Counter(metrics.NexusCompletionRequests.Name()).Record(1) + metricsHandler.Histogram(metrics.NexusCompletionLatencyHistogram.Name(), metrics.Milliseconds).Record(time.Since(startTime).Milliseconds()) + default: + metrics.NexusRequests.With(metricsHandler).Record(1) + metrics.NexusLatency.With(metricsHandler).Record(time.Since(startTime)) + if reportErr != nil { + metrics.NexusRequestErrors.With(metricsHandler).Record(1) + } + } + ti.RecordLatencyMetrics(ctx, startTime, interceptorMetricsHandler) + telemetryContext.HandleRequestError(reportErr) + }() + // recover before recording so that metrics are still recorded in case of a panic + defer func() { + recovered := recover() //nolint:revive + if recovered == nil { + return + } + err, ok := recovered.(error) + if !ok { + err = fmt.Errorf("panic: %v", recovered) + } + telemetryContext.MetricsLogger().Error("Panic captured", tag.SysStackTrace(string(debug.Stack())), tag.Error(err)) + retErr = err + }() + + return next(ctx, in) +} + func (ti *TelemetryInterceptor) RecordLatencyMetrics(ctx context.Context, startTime time.Time, metricsHandler metrics.Handler) { userLatencyDuration := time.Duration(0) if val, ok := metrics.ContextCounterGet(ctx, metrics.HistoryWorkflowExecutionCacheLatency.Name()); ok { diff --git a/common/rpc/tlsinfo/context.go b/common/rpc/tlsinfo/context.go new file mode 100644 index 00000000000..fae4c73bc5c --- /dev/null +++ b/common/rpc/tlsinfo/context.go @@ -0,0 +1,35 @@ +package tlsinfo + +import ( + "context" + "crypto/x509" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" +) + +// FromContext extracts TLS information from the context's peer value. +func FromContext(ctx context.Context) *credentials.TLSInfo { + p, ok := peer.FromContext(ctx) + if !ok { + return nil + } + if tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo); ok { + return &tlsInfo + } + return nil +} + +// PeerCert extracts an x509 certificate from given tlsInfo. +func PeerCert(tlsInfo *credentials.TLSInfo) *x509.Certificate { + if tlsInfo == nil || len(tlsInfo.State.VerifiedChains) == 0 || len(tlsInfo.State.VerifiedChains[0]) == 0 { + return nil + } + // The assumption here is that we only expect a single verified chain of certs (first[0]). + // It's unclear how we should handle a situation when more than one chain is presented, + // which subject to use. It's okay for us to limit ourselves to one chain. + // We can always extend this logic later. + // We take the first element in the chain ([0]) because that's the client cert + // (at the beginning of the chain), not intermediary CAs or the root CA (at the end of the chain). + return tlsInfo.State.VerifiedChains[0][0] +} From aec26ace5b7ca9328ddc73d1b04bd6dba3964902 Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Tue, 11 Aug 2026 18:01:54 -0700 Subject: [PATCH 03/12] adding unit tests for interceptors --- common/authorization/interceptor.go | 4 +- common/authorization/interceptor_test.go | 87 +++++++++++++++ common/rpc/interceptor/caller_info_test.go | 37 +++++++ .../interceptor/concurrent_request_limit.go | 5 +- .../concurrent_request_limit_test.go | 61 +++++++++++ .../rpc/interceptor/namespace_rate_limit.go | 10 +- .../interceptor/namespace_rate_limit_test.go | 48 +++++++++ common/rpc/interceptor/namespace_validator.go | 10 +- .../interceptor/namespace_validator_test.go | 72 +++++++++++++ common/rpc/interceptor/rate_limit.go | 10 +- common/rpc/interceptor/rate_limit_test.go | 49 +++++++++ common/rpc/interceptor/sdk_version_test.go | 56 ++++++++++ common/rpc/interceptor/telemetry_test.go | 102 ++++++++++++++++++ 13 files changed, 542 insertions(+), 9 deletions(-) diff --git a/common/authorization/interceptor.go b/common/authorization/interceptor.go index 174314212bf..b578bef2ea5 100644 --- a/common/authorization/interceptor.go +++ b/common/authorization/interceptor.go @@ -174,14 +174,14 @@ func (a *Interceptor) InterceptNexus( if err != nil { return nil, &interceptor.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, false), - Outcome: "internal_auth_error", + Outcome: "interceptor_failed", } } endpointName, err := interceptor.NexusEndpointNameFromContext(ctx) if err != nil { return nil, &interceptor.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, false), - Outcome: "internal_auth_error", + Outcome: "interceptor_failed", } } claims, _ := ctx.Value(MappedClaims).(*Claims) diff --git a/common/authorization/interceptor_test.go b/common/authorization/interceptor_test.go index e6ee83e2c7b..99218dd0db9 100644 --- a/common/authorization/interceptor_test.go +++ b/common/authorization/interceptor_test.go @@ -9,6 +9,7 @@ import ( "slices" "testing" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" commandpb "go.temporal.io/api/command/v1" @@ -22,6 +23,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/interceptor" "go.uber.org/mock/gomock" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -68,6 +70,91 @@ func TestAuthorizerInterceptorSuite(t *testing.T) { suite.Run(t, s) } +func (s *authorizerInterceptorSuite) TestInterceptNexus() { + input := interceptor.NewStartNexusOpInput( + "service", + "operation", + testNamespace, + nexus.StartOperationOptions{}, + nil, + ) + apiName, endpoint := "NexusAPI", "endpoint" + expectedTarget := &CallTarget{ + APIName: apiName, + NexusEndpointName: endpoint, + Namespace: testNamespace, + Request: input, + } + for _, tc := range []struct { + name string + ctx context.Context + authorizationResult *Result + nextCalled bool + expectedError error + }{ + { + name: "authorized", + ctx: interceptor.WithNexusEndpointName( + interceptor.WithNexusAPIName(context.Background(), apiName), + endpoint, + ), + authorizationResult: &Result{Decision: DecisionAllow}, + nextCalled: true, + }, + { + name: "unauthorized", + ctx: interceptor.WithNexusEndpointName( + interceptor.WithNexusAPIName(context.Background(), apiName), + endpoint, + ), + authorizationResult: &Result{Decision: DecisionDeny}, + expectedError: &interceptor.InterceptorError{ + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnauthorized, "permission denied"), + Outcome: "unauthorized", + }, + }, + { + name: "missing API name", + ctx: interceptor.WithNexusEndpointName(context.Background(), endpoint), + expectedError: &interceptor.InterceptorError{ + Err: errors.New("nexus API name not found in context"), + Outcome: "interceptor_failed", + }, + }, + { + name: "missing endpoint name", + ctx: interceptor.WithNexusAPIName(context.Background(), apiName), + expectedError: &interceptor.InterceptorError{ + Err: errors.New("nexus endpoint name not found in context"), + Outcome: "interceptor_failed", + }, + }, + } { + s.Run(tc.name, func() { + if tc.authorizationResult != nil { + s.mockAuthorizer.EXPECT().Authorize(gomock.Any(), nil, expectedTarget). + Return(*tc.authorizationResult, nil) + if tc.authorizationResult.Decision == DecisionDeny { + s.mockMetricsHandler.EXPECT(). + Counter(metrics.ServiceErrUnauthorizedCounter.Name()). + Return(metrics.NoopCounterMetricFunc) + } + } + + nextCalled := false + _, err := s.interceptor.InterceptNexus( + tc.ctx, + input, + func(context.Context, interceptor.NexusInterceptorInput) (any, error) { + nextCalled = true + return nil, nil + }) + s.Equal(tc.expectedError, err) + s.Equal(tc.nextCalled, nextCalled) + }) + } +} + func (s *authorizerInterceptorSuite) SetupTest() { s.Assertions = require.New(s.T()) s.controller = gomock.NewController(s.T()) diff --git a/common/rpc/interceptor/caller_info_test.go b/common/rpc/interceptor/caller_info_test.go index e028cb4266b..d808ef7bd1c 100644 --- a/common/rpc/interceptor/caller_info_test.go +++ b/common/rpc/interceptor/caller_info_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.temporal.io/api/workflowservice/v1" @@ -124,6 +125,42 @@ func (s *callerInfoSuite) TestIntercept_CallerName() { } } +func (s *callerInfoSuite) TestInterceptNexus() { + for _, tc := range []struct { + name string + input NexusInterceptorInput + callerInfo headers.CallerInfo + expectedOrigin string + }{ + { + name: "start", + input: NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + expectedOrigin: "StartNexusOperation", + }, + { + name: "cancel - preserves background origin", + input: NewCancelNexusOpInput("s", "o", testNamespace, nexus.CancelOperationOptions{}, "t"), + callerInfo: headers.SystemBackgroundHighCallerInfo, + }, + { + name: "complete", + input: NewCompleteNexusOpInput(testNamespace, nil), + expectedOrigin: "CompleteNexusOperation", + }, + } { + s.Run(tc.name, func() { + ctx := headers.SetCallerInfo(context.Background(), tc.callerInfo) + _, err := s.interceptor.InterceptNexus(ctx, tc.input, func(ctx context.Context, _ NexusInterceptorInput) (any, error) { + callerInfo := headers.GetCallerInfo(ctx) + s.Equal(testNamespace, callerInfo.CallerName) + s.Equal(tc.expectedOrigin, callerInfo.CallOrigin) + return nil, nil + }) + s.NoError(err) + }) + } +} + func (s *callerInfoSuite) TestIntercept_CallerType() { s.mockRegistry.EXPECT().GetNamespace(gomock.Any()).Return(nil, nil).AnyTimes() diff --git a/common/rpc/interceptor/concurrent_request_limit.go b/common/rpc/interceptor/concurrent_request_limit.go index cd0390e41a8..39d5f41a604 100644 --- a/common/rpc/interceptor/concurrent_request_limit.go +++ b/common/rpc/interceptor/concurrent_request_limit.go @@ -125,7 +125,10 @@ func (ni *ConcurrentRequestLimitInterceptor) InterceptNexus( ) (any, error) { apiName, err := NexusAPINameFromContext(ctx) if err != nil { - return nil, err + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, false), + Outcome: "interceptor_failed", + } } metricsHandler := GetMetricsHandlerFromContext(ctx, ni.logger) // draft-review: this looks safe to pass "in" as any, but confirm in review diff --git a/common/rpc/interceptor/concurrent_request_limit_test.go b/common/rpc/interceptor/concurrent_request_limit_test.go index e14c7159204..cc7199da9e8 100644 --- a/common/rpc/interceptor/concurrent_request_limit_test.go +++ b/common/rpc/interceptor/concurrent_request_limit_test.go @@ -2,9 +2,12 @@ package interceptor import ( "context" + "errors" "testing" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" @@ -139,6 +142,64 @@ func TestNamespaceCountLimitInterceptor_Intercept(t *testing.T) { } } +func TestConcurrentRequestLimitInterceptor_InterceptNexus(t *testing.T) { + interceptor := NewConcurrentRequestLimitInterceptor( + nil, + quotastest.NewFakeMemberCounter(1), + log.NewNoopLogger(), + dynamicconfig.GetIntPropertyFnFilteredByNamespace(1), + dynamicconfig.GetIntPropertyFnFilteredByNamespace(1), + map[string]int{"NexusAPI": 1}, + ) + input := NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) + t.Run("missing API name", func(t *testing.T) { + nextCalled := false + _, err := interceptor.InterceptNexus(context.Background(), input, func(context.Context, NexusInterceptorInput) (any, error) { + nextCalled = true + return nil, nil + }) + var interceptorErr *InterceptorError + require.ErrorAs(t, err, &interceptorErr) + require.Equal(t, "interceptor_failed", interceptorErr.Outcome) + require.False(t, nextCalled) + }) + + ctx := WithNexusAPIName(context.Background(), "NexusAPI") + + blockUntilFirstReqStarted := make(chan struct{}) + unblockFirstRequest := make(chan struct{}) + firstReqErrorCh := make(chan error, 1) + + go func() { + _, err := interceptor.InterceptNexus( + ctx, + input, + func(context.Context, NexusInterceptorInput) (any, error) { + close(blockUntilFirstReqStarted) + <-unblockFirstRequest + return nil, nil + }, + ) + firstReqErrorCh <- err + }() + <-blockUntilFirstReqStarted + // second req should never proceed to calling next + _, err := interceptor.InterceptNexus( + ctx, + input, + func(context.Context, NexusInterceptorInput) (any, error) { + t.Fatal("second request reached handler") + return nil, errors.New("throttled request reached") + }, + ) + var interceptorErr *InterceptorError + require.ErrorAs(t, err, &interceptorErr) + require.Equal(t, "namespace_concurrency_limited", interceptorErr.Outcome) + + close(unblockFirstRequest) + require.NoError(t, <-firstReqErrorCh) +} + // run the test case by simulating a bunch of blocked pollers, sending a final request, and verifying that it is either // rate limited or not. func (tc *nsCountLimitTestCase) run(t *testing.T) { diff --git a/common/rpc/interceptor/namespace_rate_limit.go b/common/rpc/interceptor/namespace_rate_limit.go index 3b86c347bb7..c72509562fa 100644 --- a/common/rpc/interceptor/namespace_rate_limit.go +++ b/common/rpc/interceptor/namespace_rate_limit.go @@ -236,11 +236,17 @@ func (ni *NamespaceRateLimitInterceptorImpl) InterceptNexus( ) (any, error) { apiName, err := NexusAPINameFromContext(ctx) if err != nil { - return nil, err + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "interceptor_failed", + } } header, err := NexusHeaderFromInterceptorInput(in) if err != nil { - return nil, err + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "interceptor_failed", + } } if err := ni.Allow(namespace.Name(in.NamespaceName()), apiName, header); err != nil { return nil, &InterceptorError{ diff --git a/common/rpc/interceptor/namespace_rate_limit_test.go b/common/rpc/interceptor/namespace_rate_limit_test.go index ab90219ccd4..11e0911c906 100644 --- a/common/rpc/interceptor/namespace_rate_limit_test.go +++ b/common/rpc/interceptor/namespace_rate_limit_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.temporal.io/api/workflowservice/v1" @@ -31,6 +32,53 @@ type namespaceRateLimitInterceptorSuite struct { mockRegistry *namespace.MockRegistry } +func (s *namespaceRateLimitInterceptorSuite) TestInterceptNexus() { + for _, tc := range []struct { + name string + apiName string + input NexusInterceptorInput + allow *bool + nextCalled bool + expectedOutcome string + }{ + {name: "allowed", apiName: "NexusOperation", input: NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), allow: new(true), nextCalled: true}, + {name: "rate limited", apiName: "NexusOperation", input: NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), allow: new(false), expectedOutcome: "namespace_rate_limited"}, + {name: "missing API name", expectedOutcome: "interceptor_failed"}, + {name: "missing request header", apiName: "NexusOperation", input: NewCompleteNexusOpInput(testNamespace, nil), expectedOutcome: "interceptor_failed"}, + } { + s.Run(tc.name, func() { + ctx := context.Background() + if tc.apiName != "" { + ctx = WithNexusAPIName(ctx, tc.apiName) + } + if tc.allow != nil { + s.mockRateLimiter.EXPECT().Allow(gomock.Any(), gomock.Any()).Return(*tc.allow) + } + input := tc.input + if input == nil { + input = NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) + } + nextCalled := false + _, err := s.newImpl(false).InterceptNexus( + ctx, + input, + func(context.Context, NexusInterceptorInput) (any, error) { + nextCalled = true + return nil, nil + }, + ) + if tc.expectedOutcome != "" { + var interceptorErr *InterceptorError + s.ErrorAs(err, &interceptorErr) + s.Equal(tc.expectedOutcome, interceptorErr.Outcome) + } else { + s.NoError(err) + } + s.Equal(tc.nextCalled, nextCalled) + }) + } +} + func TestNamespaceRateLimitInterceptorSuite(t *testing.T) { suite.Run(t, &namespaceRateLimitInterceptorSuite{}) } diff --git a/common/rpc/interceptor/namespace_validator.go b/common/rpc/interceptor/namespace_validator.go index 90a29fe7adc..54771e6b680 100644 --- a/common/rpc/interceptor/namespace_validator.go +++ b/common/rpc/interceptor/namespace_validator.go @@ -239,11 +239,17 @@ func (ni *NamespaceValidatorInterceptor) InterceptNexus( ) (any, error) { namespaceEntry, err := NexusNamespaceFromContext(ctx) if err != nil { - return nil, err + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, false), + Outcome: "interceptor_failed", + } } apiName, err := NexusAPINameFromContext(ctx) if err != nil { - return nil, err + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, false), + Outcome: "interceptor_failed", + } } if err := ni.ValidateState(namespaceEntry, apiName, in.ForwardingInfo().BusinessID); err != nil { return nil, &InterceptorError{ diff --git a/common/rpc/interceptor/namespace_validator_test.go b/common/rpc/interceptor/namespace_validator_test.go index f7599e0d44f..9017821af90 100644 --- a/common/rpc/interceptor/namespace_validator_test.go +++ b/common/rpc/interceptor/namespace_validator_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/google/uuid" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" enumspb "go.temporal.io/api/enums/v1" @@ -111,6 +112,77 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_NamespaceNotSet( } } +func (s *namespaceValidatorSuite) TestInterceptNexus() { + validator := NewNamespaceValidatorInterceptor( + s.mockRegistry, + dynamicconfig.GetBoolPropertyFn(false), + dynamicconfig.GetIntPropertyFn(100), + nil, + ) + input := NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) + for _, tc := range []struct { + name string + ctx context.Context + nextCalled bool + expectedOutcome string + }{ + { + name: "resolved namespace", + ctx: WithNexusAPIName(WithNexusNamespace(context.Background(), namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace, State: enumspb.NAMESPACE_STATE_REGISTERED}, + nil, + false, + nil, + 0, + )), api.NexusServicePrefix+"DispatchNexusTask"), + nextCalled: true, + }, + { + name: "invalid namespace state", + ctx: WithNexusAPIName(WithNexusNamespace(context.Background(), namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace, State: enumspb.NAMESPACE_STATE_DEPRECATED}, + nil, + false, + nil, + 0, + )), api.NexusServicePrefix+"DispatchNexusTask"), + expectedOutcome: "invalid_namespace_state", + }, + {name: "missing namespace", ctx: WithNexusAPIName(context.Background(), "NexusAPI"), expectedOutcome: "interceptor_failed"}, + { + name: "missing API name", + ctx: WithNexusNamespace(context.Background(), namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace, State: enumspb.NAMESPACE_STATE_REGISTERED}, + nil, + false, + nil, + 0, + )), + expectedOutcome: "interceptor_failed", + }, + } { + s.Run(tc.name, func() { + nextCalled := false + _, err := validator.InterceptNexus( + tc.ctx, + input, + func(context.Context, NexusInterceptorInput) (any, error) { + nextCalled = true + return nil, nil + }, + ) + if tc.expectedOutcome != "" { + var interceptorErr *InterceptorError + s.ErrorAs(err, &interceptorErr) + s.Equal(tc.expectedOutcome, interceptorErr.Outcome) + } else { + s.NoError(err) + } + s.Equal(tc.nextCalled, nextCalled) + }) + } +} + func (s *namespaceValidatorSuite) Test_StateValidationIntercept_NamespaceNotFound() { nvi := NewNamespaceValidatorInterceptor( diff --git a/common/rpc/interceptor/rate_limit.go b/common/rpc/interceptor/rate_limit.go index 132abd6d620..785ff499b97 100644 --- a/common/rpc/interceptor/rate_limit.go +++ b/common/rpc/interceptor/rate_limit.go @@ -100,11 +100,17 @@ func (i *RateLimitInterceptor) InterceptNexus( ) (any, error) { apiName, err := NexusAPINameFromContext(ctx) if err != nil { - return nil, err + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "interceptor_failed", + } } header, err := NexusHeaderFromInterceptorInput(in) if err != nil { - return nil, err + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "interceptor_failed", + } } if err := i.Allow(apiName, header); err != nil { return nil, &InterceptorError{ diff --git a/common/rpc/interceptor/rate_limit_test.go b/common/rpc/interceptor/rate_limit_test.go index 0c645c0be50..db04f608644 100644 --- a/common/rpc/interceptor/rate_limit_test.go +++ b/common/rpc/interceptor/rate_limit_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.temporal.io/server/common/quotas" @@ -26,6 +27,54 @@ func TestRateLimitInterceptorSuite(t *testing.T) { suite.Run(t, &rateLimitInterceptorSuite{}) } +func (s *rateLimitInterceptorSuite) TestInterceptNexus() { + for _, tc := range []struct { + name string + apiName string + input NexusInterceptorInput + allow *bool + nextCalled bool + expectedOutcome string + }{ + {name: "allowed", apiName: "NexusOperation", input: NewStartNexusOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil), allow: new(true), nextCalled: true}, + {name: "rate limited", apiName: "NexusOperation", input: NewStartNexusOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil), allow: new(false), expectedOutcome: "global_rate_limited"}, + {name: "missing API name", expectedOutcome: "interceptor_failed"}, + {name: "missing request header", apiName: "NexusOperation", input: NewCompleteNexusOpInput(testNamespace, nil), expectedOutcome: "interceptor_failed"}, + } { + s.Run(tc.name, func() { + ctx := context.Background() + interceptor := NewRateLimitInterceptor(s.mockRateLimiter, nil) + if tc.apiName != "" { + ctx = WithNexusAPIName(ctx, tc.apiName) + } + if tc.allow != nil { + s.mockRateLimiter.EXPECT().Allow(gomock.Any(), gomock.Any()).Return(*tc.allow) + } + input := tc.input + if input == nil { + input = NewStartNexusOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil) + } + nextCalled := false + _, err := interceptor.InterceptNexus( + ctx, + input, + func(context.Context, NexusInterceptorInput) (any, error) { + nextCalled = true + return nil, nil + }, + ) + if tc.expectedOutcome != "" { + var interceptorErr *InterceptorError + s.ErrorAs(err, &interceptorErr) + s.Equal(tc.expectedOutcome, interceptorErr.Outcome) + } else { + s.NoError(err) + } + s.Equal(tc.nextCalled, nextCalled) + }) + } +} + func (s *rateLimitInterceptorSuite) SetupTest() { s.Assertions = require.New(s.T()) s.controller = gomock.NewController(s.T()) diff --git a/common/rpc/interceptor/sdk_version_test.go b/common/rpc/interceptor/sdk_version_test.go index 25ed973100e..2376ba6a97b 100644 --- a/common/rpc/interceptor/sdk_version_test.go +++ b/common/rpc/interceptor/sdk_version_test.go @@ -5,7 +5,9 @@ import ( "sort" "testing" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/versioninfo" ) @@ -64,3 +66,57 @@ func TestSDKVersionRecorder(t *testing.T) { assert.Equal(t, headers.ClientNameTypeScriptSDK, info[1].Name) assert.Equal(t, sdkVersion, info[1].Version) } + +func TestSDKVersionInterceptNexus(t *testing.T) { + clientVersion := "1.10.1" + for _, tc := range []struct { + name string + ctx context.Context + expectedOutcome string + }{ + { + name: "supported client", + ctx: headers.SetVersionsForTests( + context.Background(), + clientVersion, + headers.ClientNameGoSDK, + headers.SupportedServerVersions, + headers.AllFeatures, + ), + }, + { + name: "unsupported client", + ctx: headers.SetVersionsForTests( + context.Background(), + "unparseable.client.version", + headers.ClientNameGoSDK, + headers.SupportedServerVersions, + headers.AllFeatures, + ), + expectedOutcome: "unsupported_client", + }, + } { + t.Run(tc.name, func(t *testing.T) { + interceptor := NewSDKVersionInterceptor() + nextCalled := false + _, err := interceptor.InterceptNexus( + tc.ctx, + NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + func(context.Context, NexusInterceptorInput) (any, error) { + nextCalled = true + return nil, nil + }, + ) + if tc.expectedOutcome != "" { + var interceptorErr *InterceptorError + require.ErrorAs(t, err, &interceptorErr) + require.Equal(t, tc.expectedOutcome, interceptorErr.Outcome) + require.False(t, nextCalled) + } else { + require.True(t, nextCalled) + require.NoError(t, err) + require.Contains(t, interceptor.GetAndResetSDKInfo(), versioninfo.SDKInfo{Name: headers.ClientNameGoSDK, Version: clientVersion}) + } + }) + } +} diff --git a/common/rpc/interceptor/telemetry_test.go b/common/rpc/interceptor/telemetry_test.go index c57f9efb1dd..6b15046e1db 100644 --- a/common/rpc/interceptor/telemetry_test.go +++ b/common/rpc/interceptor/telemetry_test.go @@ -2,9 +2,12 @@ package interceptor import ( "context" + "errors" "testing" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -26,6 +29,105 @@ import ( "google.golang.org/grpc/status" ) +type nexusTelemetryContext struct { + outcome string + handled error + hasHandled bool +} + +func (c *nexusTelemetryContext) MetricsHandler(error) metrics.Handler { + return metrics.NoopMetricsHandler +} + +func (c *nexusTelemetryContext) MetricsHandlerForInterceptors() metrics.Handler { + return metrics.NoopMetricsHandler +} + +func (c *nexusTelemetryContext) MetricsLogger() log.Logger { + return log.NewNoopLogger() +} + +func (c *nexusTelemetryContext) SetMetricsOutcome(outcome string) { + c.outcome = outcome +} + +func (c *nexusTelemetryContext) SetFailureSource(string) {} + +func (c *nexusTelemetryContext) HandleRequestError(err error) { + c.handled = err + c.hasHandled = true +} + +func TestTelemetryInterceptNexus(t *testing.T) { + input := NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) + for _, tc := range []struct { + name string + setContext bool + handler NexusHandlerFunc + expectedOutcome string + expectedError error + nextCalled bool + expectHandled bool + }{ + { + name: "regular telemetry capture", + setContext: true, + handler: func(context.Context, NexusInterceptorInput) (any, error) { + return nil, nil + }, + nextCalled: true, + expectHandled: true, + }, + { + name: "missing telemetry context", + handler: func(context.Context, NexusInterceptorInput) (any, error) { + return nil, nil + }, + expectedError: errors.New("telemetry context not found"), + }, + { + name: "tagged error", + setContext: true, + handler: func(context.Context, NexusInterceptorInput) (any, error) { + return nil, &InterceptorError{Err: errors.New("rejected"), Outcome: "rejected"} + }, + expectedOutcome: "rejected", + expectedError: &InterceptorError{Err: errors.New("rejected"), Outcome: "rejected"}, + nextCalled: true, + expectHandled: true, + }, + { + name: "ensure metrics still captured on panics", + setContext: true, + handler: func(context.Context, NexusInterceptorInput) (any, error) { + panic("") + }, + expectedError: errors.New("panic: "), + nextCalled: true, + expectHandled: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + telemetryContext := &nexusTelemetryContext{} + if tc.setContext { + ctx = WithTelemetryContext(ctx, telemetryContext) + } + nextCalled := false + _, err := (&TelemetryInterceptor{}).InterceptNexus(ctx, input, func(ctx context.Context, input NexusInterceptorInput) (any, error) { + nextCalled = true + return tc.handler(ctx, input) + }) + require.Equal(t, tc.expectedError, err) + require.Equal(t, tc.nextCalled, nextCalled) + if tc.setContext { + require.Equal(t, tc.expectedOutcome, telemetryContext.outcome) + } + require.Equal(t, tc.expectHandled, telemetryContext.hasHandled) + }) + } +} + const ( startWorkflow = "StartWorkflowExecution" executeMultiOps = "ExecuteMultiOperation" From 0127976082ea7850ec6e36354f7b7c2e14c7ae7b Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Mon, 10 Aug 2026 17:33:38 -0700 Subject: [PATCH 04/12] add forward interceptor for Nexus --- service/frontend/nexus_forward_interceptor.go | 301 ++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 service/frontend/nexus_forward_interceptor.go diff --git a/service/frontend/nexus_forward_interceptor.go b/service/frontend/nexus_forward_interceptor.go new file mode 100644 index 00000000000..3db1fdae060 --- /dev/null +++ b/service/frontend/nexus_forward_interceptor.go @@ -0,0 +1,301 @@ +package frontend + +import ( + "context" + "errors" + "net/http" + "net/http/httptrace" + "net/url" + "strconv" + "time" + + "github.com/nexus-rpc/sdk-go/nexus" + "go.temporal.io/server/common" + "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/namespace" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/common/rpc/interceptor" +) + +type nexusForwardingInterceptor struct { + logger log.Logger + clusterMetadata cluster.Metadata + redirectionInterceptor *interceptor.Redirection + forwardingClients *cluster.FrontendHTTPClientCache + serviceConfig *Config + httpTraceProvider commonnexus.HTTPClientTraceProvider +} + +func newNexusForwardingInterceptor( + logger log.Logger, + clusterMetadata cluster.Metadata, + redirectionInterceptor *interceptor.Redirection, + forwardingClients *cluster.FrontendHTTPClientCache, + serviceConfig *Config, + httpTraceProvider commonnexus.HTTPClientTraceProvider, +) *nexusForwardingInterceptor { + return &nexusForwardingInterceptor{ + logger: logger, + clusterMetadata: clusterMetadata, + redirectionInterceptor: redirectionInterceptor, + forwardingClients: forwardingClients, + serviceConfig: serviceConfig, + httpTraceProvider: httpTraceProvider, + } +} + +func (i *nexusForwardingInterceptor) InterceptNexus( + ctx context.Context, + in interceptor.NexusInterceptorInput, + next interceptor.NexusHandlerFunc, +) (out any, retErr error) { + info := in.ForwardingInfo() + header, err := interceptor.NexusHeaderFromInterceptorInput(in) + if err != nil { + return nil, err + } + namespaceEntry, err := interceptor.NexusNamespaceFromContext(ctx) + if err != nil { + return nil, err + } + currentCluster := i.clusterMetadata.GetCurrentClusterName() + targetCluster := namespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: info.BusinessID}) + if !namespaceEntry.IsGlobalNamespace() || targetCluster == currentCluster { + return next(ctx, in) + } + if !i.shouldForwardRequest(ctx, header, namespaceEntry) { + return nil, &interceptor.InterceptorError{ + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive"), + Outcome: "namespace_inactive_forwarding_disabled", + } + } + + telemetryContext, err := interceptor.TelemetryContextFromContext(ctx) + if err != nil { + return nil, err + } + telemetryContext.SetMetricsOutcome("request_forwarded") + + metricsHandler, forwardStartTime := i.redirectionInterceptor.BeforeCall(interceptor.NexusMethodName(in)) + defer func() { + redirectionErr := retErr + if taggedErr, ok := errors.AsType[*interceptor.InterceptorError](retErr); ok { + redirectionErr = taggedErr.Err + } + i.redirectionInterceptor.AfterCall(metricsHandler, forwardStartTime, targetCluster, namespaceEntry.Name().String(), redirectionErr) + }() + + switch request := in.(type) { + case interceptor.StartNexusOpInput: + out, retErr = i.forwardStartOperation(ctx, request, info, namespaceEntry, targetCluster, telemetryContext) + case interceptor.CancelNexusOpInput: + retErr = i.forwardCancelOperation(ctx, request, info, namespaceEntry, targetCluster, telemetryContext) + case interceptor.CompleteNexusOpInput: + retErr = i.forwardCompleteOperation(ctx, request, info, namespaceEntry, targetCluster, telemetryContext) + default: + return nil, &interceptor.InterceptorError{ + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "forwarding failed, unknown operation type"), + } + } + return out, retErr +} + +func (i *nexusForwardingInterceptor) shouldForwardRequest( + ctx context.Context, + header headers.HeaderGetter, + namespaceEntry *namespace.Namespace, +) bool { + redirectAllowed, err := strconv.ParseBool(header.Get(interceptor.DCRedirectionContextHeaderName)) + if err != nil { + redirectAllowed = true + } + return redirectAllowed && + i.redirectionInterceptor.RedirectionAllowed(ctx) && + namespaceEntry.IsGlobalNamespace() && + i.serviceConfig.EnableNamespaceNotActiveAutoForwarding(namespaceEntry.Name().String()) +} + +func (i *nexusForwardingInterceptor) forwardStartOperation( + ctx context.Context, + request interceptor.StartNexusOpInput, + info interceptor.NexusForwardingInfo, + namespaceEntry *namespace.Namespace, + targetCluster string, + telemetryContext interceptor.TelemetryContext, +) (any, error) { + request.StartOperationOptions.Header[interceptor.DCRedirectionAPIHeaderName] = "true" + request.StartOperationOptions.Header[interceptor.DCRedirectionSourceCellHeaderName] = i.clusterMetadata.GetCurrentClusterName() + client, err := i.nexusClientForActiveCluster(request.ServiceName(), info, namespaceEntry, targetCluster, telemetryContext) + if err != nil { + return nil, err + } + ctx = i.withForwardingTrace(ctx, "StartNexusOperation", request.OperationName(), request.StartOperationOptions.RequestID, info, namespaceEntry, targetCluster) + response, err := client.StartOperation(ctx, request.OperationName(), request.StartOperationInput.Reader, request.StartOperationOptions) + if err != nil { + i.logger.Error("received error from remote cluster for forwarded Nexus start operation request", tag.Error(err)) + return nil, &interceptor.InterceptorError{Err: err, Outcome: "forwarded_request_error"} + } + if response.Successful != nil { + return &nexus.HandlerStartOperationResultSync[any]{Value: response.Successful.Reader}, nil + } + return &nexus.HandlerStartOperationResultAsync{OperationToken: response.Pending.Token}, nil +} + +func (i *nexusForwardingInterceptor) forwardCancelOperation( + ctx context.Context, + request interceptor.CancelNexusOpInput, + info interceptor.NexusForwardingInfo, + namespaceEntry *namespace.Namespace, + targetCluster string, + telemetryContext interceptor.TelemetryContext, +) error { + request.CancelOperationOptions.Header[interceptor.DCRedirectionAPIHeaderName] = "true" + request.CancelOperationOptions.Header[interceptor.DCRedirectionSourceCellHeaderName] = i.clusterMetadata.GetCurrentClusterName() + client, err := i.nexusClientForActiveCluster(request.ServiceName(), info, namespaceEntry, targetCluster, telemetryContext) + if err != nil { + return err + } + handle, err := client.NewOperationHandle(request.OperationName(), request.CancellationToken) + if err != nil { + i.logger.Warn("invalid Nexus cancel operation", tag.Error(err)) + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation") + } + ctx = i.withForwardingTrace(ctx, "CancelNexusOperation", request.OperationName(), "", info, namespaceEntry, targetCluster) + if err := handle.Cancel(ctx, request.CancelOperationOptions); err != nil { + i.logger.Error("received error from remote cluster for forwarded Nexus cancel operation request", tag.Error(err)) + return &interceptor.InterceptorError{Err: err, Outcome: "forwarded_request_error"} + } + return nil +} + +func (i *nexusForwardingInterceptor) forwardCompleteOperation( + ctx context.Context, + request interceptor.CompleteNexusOpInput, + info interceptor.NexusForwardingInfo, + namespaceEntry *namespace.Namespace, + targetCluster string, + telemetryContext interceptor.TelemetryContext, +) error { + client, err := i.forwardingClients.Get(targetCluster) + if err != nil { + i.logger.Error("unable to get HTTP client for forward request", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) + return &interceptor.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} + } + forwardURL, err := url.JoinPath(client.BaseURL(), commonnexus.RouteCompletionCallback.Path(namespaceEntry.Name().String())) + if err != nil { + i.logger.Error("failed to construct forwarding request URL", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.TargetCluster(targetCluster)) + return &interceptor.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} + } + request.CompletionRequest.HTTPRequest.Header.Set(interceptor.DCRedirectionAPIHeaderName, "true") + request.CompletionRequest.HTTPRequest.Header.Set(interceptor.DCRedirectionSourceCellHeaderName, i.clusterMetadata.GetCurrentClusterName()) + info.OriginalRequestHeaders.Set(interceptor.DCRedirectionAPIHeaderName, "true") + info.OriginalRequestHeaders.Set(interceptor.DCRedirectionSourceCellHeaderName, i.clusterMetadata.GetCurrentClusterName()) + completion, err := completeOperationOptions(request.CompletionRequest) + if err != nil { + return err + } + ctx = i.withForwardingTrace(ctx, "CompleteNexusOperation", "", "", info, namespaceEntry, targetCluster) + err = nexusrpc.NewCompletionHTTPClient(nexusrpc.CompletionHTTPClientOptions{ + HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: client, originalRequestHeaders: info.OriginalRequestHeaders, telemetryContext: telemetryContext}).Do, + }).CompleteOperation(ctx, forwardURL, completion) + if err != nil { + return &interceptor.InterceptorError{Err: err, Outcome: "forwarded_request_error"} + } + return nil +} + +func completeOperationOptions(request *nexusrpc.CompletionRequest) (nexusrpc.CompleteOperationOptions, error) { + switch request.State { + case nexus.OperationStateSucceeded: + return nexusrpc.CompleteOperationOptions{Result: request.Result.Reader, OperationToken: request.OperationToken, StartTime: request.StartTime, CloseTime: request.CloseTime, Links: request.Links}, nil + case nexus.OperationStateFailed, nexus.OperationStateCanceled: + return nexusrpc.CompleteOperationOptions{Error: request.Error, OperationToken: request.OperationToken, StartTime: request.StartTime, CloseTime: request.CloseTime, Links: request.Links}, nil + default: + return nexusrpc.CompleteOperationOptions{}, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation state: %q", request.State) + } +} + +func (i *nexusForwardingInterceptor) nexusClientForActiveCluster( + service string, + info interceptor.NexusForwardingInfo, + namespaceEntry *namespace.Namespace, + targetCluster string, + telemetryContext interceptor.TelemetryContext, +) (*nexusrpc.HTTPClient, error) { + httpClient, err := i.forwardingClients.Get(targetCluster) + if err != nil { + i.logger.Error("failed to forward Nexus request: error creating HTTP client", tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) + return nil, &interceptor.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} + } + var baseURL string + if i.serviceConfig.NexusForwardRequestUseEndpoint() && info.EndpointID != "" { + baseURL, err = url.JoinPath(httpClient.BaseURL(), commonnexus.RouteDispatchNexusTaskByEndpoint.Path(info.EndpointID)) + } else { + baseURL, err = url.JoinPath(httpClient.BaseURL(), commonnexus.RouteDispatchNexusTaskByNamespaceAndTaskQueue.Path(commonnexus.NamespaceAndTaskQueue{Namespace: namespaceEntry.Name().String(), TaskQueue: info.TaskQueue})) + } + if err != nil { + i.logger.Error("failed to forward Nexus request: error constructing ServiceBaseURL", tag.URL(httpClient.BaseURL()), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.WorkflowTaskQueueName(info.TaskQueue), tag.Error(err)) + return nil, &interceptor.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} + } + return nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ + HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: httpClient, originalRequestHeaders: info.OriginalRequestHeaders, telemetryContext: telemetryContext}).Do, + BaseURL: baseURL, + Service: service, + }) +} + +func (i *nexusForwardingInterceptor) withForwardingTrace( + ctx context.Context, + method string, + operation string, + requestID string, + info interceptor.NexusForwardingInfo, + namespaceEntry *namespace.Namespace, + targetCluster string, +) context.Context { + if i.httpTraceProvider == nil { + return ctx + } + traceLogger := log.With(i.logger, + tag.Operation(method), + tag.WorkflowNamespace(namespaceEntry.Name().String()), + tag.RequestID(requestID), + tag.NexusOperation(operation), + tag.Endpoint(info.EndpointName), + tag.AttemptStart(time.Now().UTC()), + tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), + tag.TargetCluster(targetCluster), + ) + if trace := i.httpTraceProvider.NewForwardingTrace(traceLogger); trace != nil { + return httptrace.WithClientTrace(ctx, trace) + } + return ctx +} + +type nexusForwardingHTTPHeaderWrapper struct { + client *common.FrontendHTTPClient + originalRequestHeaders http.Header + telemetryContext interceptor.TelemetryContext +} + +func (f *nexusForwardingHTTPHeaderWrapper) Do(request *http.Request) (*http.Response, error) { + // for forwarded requests, copy the original HTTP headers without sanitization. + for name, values := range f.originalRequestHeaders { + if request.Header.Get(name) == "" { + request.Header.Set(name, values[0]) + } + } + response, err := f.client.Do(request) + if err != nil { + return nil, err + } + + if source := response.Header.Get(commonnexus.FailureSourceHeaderName); source != "" && f.telemetryContext != nil { + f.telemetryContext.SetFailureSource(source) + } + return response, nil +} From 1d44ef1f56865505f9778eb18e72a8d4177b64de Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Wed, 12 Aug 2026 14:22:14 -0700 Subject: [PATCH 05/12] add tests for forwarder --- service/frontend/nexus_forward_interceptor.go | 21 +- .../nexus_forward_interceptor_test.go | 209 ++++++++++++++++++ 2 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 service/frontend/nexus_forward_interceptor_test.go diff --git a/service/frontend/nexus_forward_interceptor.go b/service/frontend/nexus_forward_interceptor.go index 3db1fdae060..3865e576c2d 100644 --- a/service/frontend/nexus_forward_interceptor.go +++ b/service/frontend/nexus_forward_interceptor.go @@ -25,11 +25,15 @@ type nexusForwardingInterceptor struct { logger log.Logger clusterMetadata cluster.Metadata redirectionInterceptor *interceptor.Redirection - forwardingClients *cluster.FrontendHTTPClientCache + forwardingClients frontendHTTPClientCache serviceConfig *Config httpTraceProvider commonnexus.HTTPClientTraceProvider } +type frontendHTTPClientCache interface { + Get(targetClusterName string) (*common.FrontendHTTPClient, error) +} + func newNexusForwardingInterceptor( logger log.Logger, clusterMetadata cluster.Metadata, @@ -56,11 +60,17 @@ func (i *nexusForwardingInterceptor) InterceptNexus( info := in.ForwardingInfo() header, err := interceptor.NexusHeaderFromInterceptorInput(in) if err != nil { - return nil, err + return nil, &interceptor.InterceptorError{ + Err: err, + Outcome: "interceptor_failed", + } } namespaceEntry, err := interceptor.NexusNamespaceFromContext(ctx) if err != nil { - return nil, err + return nil, &interceptor.InterceptorError{ + Err: err, + Outcome: "interceptor_failed", + } } currentCluster := i.clusterMetadata.GetCurrentClusterName() targetCluster := namespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: info.BusinessID}) @@ -76,7 +86,10 @@ func (i *nexusForwardingInterceptor) InterceptNexus( telemetryContext, err := interceptor.TelemetryContextFromContext(ctx) if err != nil { - return nil, err + return nil, &interceptor.InterceptorError{ + Err: err, + Outcome: "interceptor_failed", + } } telemetryContext.SetMetricsOutcome("request_forwarded") diff --git a/service/frontend/nexus_forward_interceptor_test.go b/service/frontend/nexus_forward_interceptor_test.go new file mode 100644 index 00000000000..4e4df35dc43 --- /dev/null +++ b/service/frontend/nexus_forward_interceptor_test.go @@ -0,0 +1,209 @@ +package frontend + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/cluster/clustertest" + "go.temporal.io/server/common/config" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/interceptor" +) + +func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { + metadata := clustertest.NewMetadataForTest(cluster.NewTestClusterMetadataConfig(true, true)) + currentCluster := cluster.TestCurrentClusterName + remoteCluster := cluster.TestAlternativeClusterName + + type requestDisposition int + const ( + requestFailed requestDisposition = iota + requestHandledLocally + requestForwarded + ) + + // dummy server to simulate fowarded req + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprint(w, `{"token":"operation-token","state":"running"}`) + })) + defer server.Close() + forwardingClient := testFrontendHTTPClientCache{clients: map[string]*common.FrontendHTTPClient{ + remoteCluster: { + Client: *server.Client(), + Address: server.Listener.Addr().String(), + Scheme: "http", + }, + }} + input := interceptor.NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{ + Header: nexus.Header{"X-Request": "request"}, + }, nexus.NewLazyValue(nexus.DefaultSerializer(), &nexus.Reader{ + ReadCloser: io.NopCloser(bytes.NewBufferString(`"input"`)), + Header: nexus.Header{"type": "json"}, + })) + input.WithForwardingInfo(interceptor.NexusForwardingInfo{ + OriginalRequestHeaders: http.Header{"X-Original": {"original"}}, + TaskQueue: "task-queue", + }) + + for _, tc := range []struct { + name string + namespace *namespace.Namespace + forwardingOn bool + expectedOutcome string + disposition requestDisposition + }{ + { + name: "local namespace should resolve", + namespace: namespace.NewLocalNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace}, + nil, + currentCluster, + ), + disposition: requestHandledLocally, + }, + { + name: "global namespace with forwarding enabled should redirect", + namespace: namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace}, + nil, + true, + &persistencespb.NamespaceReplicationConfig{ActiveClusterName: remoteCluster, Clusters: []string{currentCluster, remoteCluster}}, + 0, + ), + forwardingOn: true, + disposition: requestForwarded, + }, + { + name: "global namespace with forwarding disabled should fail", + namespace: namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace}, + nil, + true, + &persistencespb.NamespaceReplicationConfig{ActiveClusterName: remoteCluster, Clusters: []string{currentCluster, remoteCluster}}, + 0, + ), + forwardingOn: false, + expectedOutcome: "namespace_inactive_forwarding_disabled", + disposition: requestFailed, + }, + { + name: "global namespace with forwarding enabled to unknown cluster fails", + namespace: namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace}, + nil, + true, + &persistencespb.NamespaceReplicationConfig{ActiveClusterName: "unknown-cluster", Clusters: []string{currentCluster}}, + 0, + ), + forwardingOn: true, + expectedOutcome: "request_forwarding_failed", + disposition: requestFailed, + }, + } { + t.Run(tc.name, func(t *testing.T) { + forwarder := &nexusForwardingInterceptor{ + logger: log.NewNoopLogger(), + clusterMetadata: metadata, + forwardingClients: forwardingClient, + redirectionInterceptor: interceptor.NewRedirection( + dynamicconfig.GetBoolPropertyFnFilteredByNamespace(true), + dynamicconfig.GetBoolPropertyFnFilteredByNamespace(false), + nil, + config.DCRedirectionPolicy{Policy: interceptor.DCRedirectionPolicyAllAPIsForwarding}, + log.NewNoopLogger(), + nil, + metrics.NoopMetricsHandler, + clock.NewRealTimeSource(), + metadata, + ), + serviceConfig: &Config{ + EnableNamespaceNotActiveAutoForwarding: dynamicconfig.GetBoolPropertyFnFilteredByNamespace(tc.forwardingOn), + NexusForwardRequestUseEndpoint: dynamicconfig.GetBoolPropertyFn(false), + }, + } + ctx := interceptor.WithNexusNamespace(context.Background(), tc.namespace) + ctx = interceptor.WithTelemetryContext(ctx, &forwardingTelemetryContext{}) + nextCalled := false + result, err := forwarder.InterceptNexus( + ctx, + input, + func(context.Context, interceptor.NexusInterceptorInput) (any, error) { + nextCalled = true + return requestHandledLocally, nil + }, + ) + if tc.expectedOutcome != "" { + var interceptorErr *interceptor.InterceptorError + require.ErrorAs(t, err, &interceptorErr) + require.Equal(t, tc.expectedOutcome, interceptorErr.Outcome) + } else { + require.NoError(t, err) + } + expectedNextCalled := tc.disposition == requestHandledLocally + require.Equal(t, expectedNextCalled, nextCalled) + switch tc.disposition { + case requestHandledLocally: + require.Equal(t, requestHandledLocally, result) + case requestForwarded: + require.IsType(t, &nexus.HandlerStartOperationResultAsync{}, result) + case requestFailed: + require.Nil(t, result) + default: + t.Fatal("unexpected disposition") + } + }) + } +} + +type forwardingTelemetryContext struct { + failureSource string +} + +func (*forwardingTelemetryContext) MetricsHandler(error) metrics.Handler { + return metrics.NoopMetricsHandler +} + +func (*forwardingTelemetryContext) MetricsHandlerForInterceptors() metrics.Handler { + return metrics.NoopMetricsHandler +} + +func (*forwardingTelemetryContext) MetricsLogger() log.Logger { + return log.NewNoopLogger() +} + +func (*forwardingTelemetryContext) SetMetricsOutcome(string) {} + +func (c *forwardingTelemetryContext) SetFailureSource(source string) { + c.failureSource = source +} + +func (*forwardingTelemetryContext) HandleRequestError(error) {} + +type testFrontendHTTPClientCache struct { + clients map[string]*common.FrontendHTTPClient +} + +func (c testFrontendHTTPClientCache) Get(clusterName string) (*common.FrontendHTTPClient, error) { + client, ok := c.clients[clusterName] + if !ok { + return nil, errors.New("unknown cluster") + } + return client, nil +} From 758510e36ff245a8edded6be9633329a2d2e59fc Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Mon, 10 Aug 2026 17:35:39 -0700 Subject: [PATCH 06/12] adapting handlers to new Nexus frontend interceptors --- common/authorization/interceptor.go | 6 +- common/rpc/interceptor/nexus.go | 16 +- service/frontend/fx.go | 1 + .../frontend/nexus_completion_http_handler.go | 414 ++++++----------- service/frontend/nexus_handler.go | 438 +++++++----------- service/frontend/nexus_handler_test.go | 208 --------- .../frontend/nexus_operation_http_handler.go | 42 +- temporal/fx.go | 6 + temporal/server_option.go | 12 + temporal/server_options.go | 2 + 10 files changed, 367 insertions(+), 778 deletions(-) diff --git a/common/authorization/interceptor.go b/common/authorization/interceptor.go index b578bef2ea5..452713f0feb 100644 --- a/common/authorization/interceptor.go +++ b/common/authorization/interceptor.go @@ -184,20 +184,18 @@ func (a *Interceptor) InterceptNexus( Outcome: "interceptor_failed", } } - claims, _ := ctx.Value(MappedClaims).(*Claims) - var req any + claims, _ := ctx.Value(MappedClaims).(*Claims) //nolint:revive // unchecked-type-assertion: empty claims will 403 // draft-review: check if this might be required to preserve compatibility for custom authorizers // or if its ok since an interface was not already used instead // switch in.(type) { // case interceptor.StartNexusOpInput, interceptor.CancelNexusOpInput: // case *interceptor.CancelNexusOpInput: // } - req = in ct := &CallTarget{ APIName: apiName, NexusEndpointName: endpointName, Namespace: namespaceName, - Request: req, + Request: in, } principal, err := a.Authorize(ctx, claims, ct) if err != nil { diff --git a/common/rpc/interceptor/nexus.go b/common/rpc/interceptor/nexus.go index c0ddbcf4f13..4570485c3dc 100644 --- a/common/rpc/interceptor/nexus.go +++ b/common/rpc/interceptor/nexus.go @@ -90,7 +90,7 @@ func NexusHeaderFromInterceptorInput(in NexusInterceptorInput) (headers.HeaderGe return opts.CancelOperationOptions.Header, nil case CompleteNexusOpInput: if opts.CompletionRequest == nil || opts.CompletionRequest.HTTPRequest == nil { - return nil, errors.New("Nexus completion request not found") + return nil, errors.New("nexus completion request not found") } return opts.CompletionRequest.HTTPRequest.Header, nil default: @@ -99,6 +99,8 @@ func NexusHeaderFromInterceptorInput(in NexusInterceptorInput) (headers.HeaderGe } // draft-review: verify that these are the "right" methods/names +// +//nolint:staticcheck func NexusMethodName(in NexusInterceptorInput) string { switch in.(type) { case StartNexusOpInput: @@ -166,6 +168,8 @@ type CompleteNexusOpInput struct { } // draft-review: Complete doesnt need servicename/op - verify +// +//nolint:staticcheck func NewCompleteNexusOpInput( namespaceName string, request *nexusrpc.CompletionRequest, @@ -202,7 +206,7 @@ func WithNexusAPIName(ctx context.Context, apiName string) context.Context { func NexusAPINameFromContext(ctx context.Context) (string, error) { apiName, ok := ctx.Value(nexusAPINameContextKey{}).(string) if !ok { - return "", errors.New("Nexus API name not found in context") + return "", errors.New("nexus API name not found in context") } return apiName, nil } @@ -215,7 +219,7 @@ func WithNexusEndpointName(ctx context.Context, endpointName string) context.Con func NexusEndpointNameFromContext(ctx context.Context) (string, error) { endpointName, ok := ctx.Value(nexusEndpointNameContextKey{}).(string) if !ok { - return "", errors.New("Nexus endpoint name not found in context") + return "", errors.New("nexus endpoint name not found in context") } return endpointName, nil } @@ -225,11 +229,13 @@ func WithNexusNamespace(ctx context.Context, namespaceEntry *namespace.Namespace return context.WithValue(ctx, nexusNamespaceContextKey{}, namespaceEntry) } -// darft-review: ideally, there is some utility to lookup by name -> Namespace +// draft-review: ideally, there is some utility to lookup by name -> Namespace +// +//nolint:staticcheck func NexusNamespaceFromContext(ctx context.Context) (*namespace.Namespace, error) { namespaceEntry, ok := ctx.Value(nexusNamespaceContextKey{}).(*namespace.Namespace) if !ok { - return nil, errors.New("Nexus namespace not found in context") + return nil, errors.New("nexus namespace not found in context") } return namespaceEntry, nil } diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 2ba531cf795..424c17941c9 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -130,6 +130,7 @@ var Module = fx.Options( fx.Provide(OperatorHandlerProvider), fx.Provide(NewVersionChecker), fx.Provide(ServiceResolverProvider), + fx.Provide(newNexusForwardingInterceptor), fx.Provide(newNexusCompletionHandler), fx.Provide(NewNexusOperationHTTPHandler), fx.Provide(newNexusCompletionHTTPHandler), diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index 66b323612ba..235e70482b3 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -3,12 +3,8 @@ package frontend import ( "context" "errors" - "fmt" "net/http" - "net/http/httptrace" "net/url" - "runtime/debug" - "strconv" "strings" "time" @@ -18,7 +14,6 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/server/api/historyservice/v1" tokenspb "go.temporal.io/server/api/token/v1" - "go.temporal.io/server/common" "go.temporal.io/server/common/authorization" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/headers" @@ -43,25 +38,27 @@ const nexusCompletionAPIName = configs.CompleteNexusOperation const nexusCompletionMethodName = "CompleteNexusOperation" type nexusCompletionHandler struct { - ClusterMetadata cluster.Metadata - NamespaceRegistry namespace.Registry - Logger log.Logger - MetricsHandler metrics.Handler - Config *Config - CallbackTokenGenerator *commonnexus.CallbackTokenGenerator - HistoryClient resource.HistoryClient - TelemetryInterceptor *interceptor.TelemetryInterceptor - RequestErrorHandler *interceptor.RequestErrorHandler - NamespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor - NamespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor - NamespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor - RateLimitInterceptor *interceptor.RateLimitInterceptor - AuthInterceptor *authorization.Interceptor - RedirectionInterceptor *interceptor.Redirection - ForwardingClients *cluster.FrontendHTTPClientCache - HTTPTraceProvider commonnexus.HTTPClientTraceProvider - clientVersionChecker headers.VersionChecker - preProcessErrorsCounter metrics.CounterIface + ClusterMetadata cluster.Metadata + NamespaceRegistry namespace.Registry + Logger log.Logger + MetricsHandler metrics.Handler + Config *Config + CallbackTokenGenerator *commonnexus.CallbackTokenGenerator + HistoryClient resource.HistoryClient + // TelemetryInterceptor *interceptor.TelemetryInterceptor + RequestErrorHandler *interceptor.RequestErrorHandler + // NamespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor + // NamespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor + // NamespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor + // RateLimitInterceptor *interceptor.RateLimitInterceptor + AuthInterceptor *authorization.Interceptor + // RedirectionInterceptor *interceptor.Redirection + ForwardingClients *cluster.FrontendHTTPClientCache + HTTPTraceProvider commonnexus.HTTPClientTraceProvider + NexusForwarder *nexusForwardingInterceptor + nexusInterceptors []interceptor.NexusInterceptor + clientVersionChecker headers.VersionChecker + preProcessErrorsCounter metrics.CounterIface } type nexusCompletionHTTPHandler struct { @@ -86,27 +83,47 @@ func newNexusCompletionHandler( redirectionInterceptor *interceptor.Redirection, forwardingClients *cluster.FrontendHTTPClientCache, httpTraceProvider commonnexus.HTTPClientTraceProvider, + nexusForwarder *nexusForwardingInterceptor, + sdkVersionInterceptor *interceptor.SDKVersionInterceptor, + callerInfoInterceptor *interceptor.CallerInfoInterceptor, + customNexusInterceptors []interceptor.NexusInterceptor, ) *nexusCompletionHandler { + nexusInterceptors := []interceptor.NexusInterceptor{ + telemetryInterceptor.InterceptNexus, + authInterceptor.InterceptNexus, + nexusForwarder.InterceptNexus, + namespaceValidationInterceptor.InterceptNexus, + namespaceConcurrencyLimitInterceptor.InterceptNexus, + namespaceRateLimitInterceptor.InterceptNexus, + rateLimitInterceptor.InterceptNexus, + sdkVersionInterceptor.InterceptNexus, + callerInfoInterceptor.InterceptNexus, + } + // draft-review: check if the customNexusInterceptors should be in the middle of + // the chain instead. Interleaved howver, is out of scope and will not be supported + nexusInterceptors = append(nexusInterceptors, customNexusInterceptors...) return &nexusCompletionHandler{ - ClusterMetadata: clusterMetadata, - NamespaceRegistry: namespaceRegistry, - Logger: log.With(logger, tag.NexusStageCallerInbound), - MetricsHandler: metricsHandler, - Config: serviceConfig, - CallbackTokenGenerator: callbackTokenGenerator, - HistoryClient: historyClient, - TelemetryInterceptor: telemetryInterceptor, - RequestErrorHandler: requestErrorHandler, - NamespaceValidationInterceptor: namespaceValidationInterceptor, - NamespaceRateLimitInterceptor: namespaceRateLimitInterceptor, - NamespaceConcurrencyLimitInterceptor: namespaceConcurrencyLimitInterceptor, - RateLimitInterceptor: rateLimitInterceptor, - AuthInterceptor: authInterceptor, - RedirectionInterceptor: redirectionInterceptor, - ForwardingClients: forwardingClients, - HTTPTraceProvider: httpTraceProvider, - clientVersionChecker: headers.NewDefaultVersionChecker(), - preProcessErrorsCounter: metricsHandler.Counter(metrics.NexusCompletionRequestPreProcessErrors.Name()), + ClusterMetadata: clusterMetadata, + NamespaceRegistry: namespaceRegistry, + Logger: log.With(logger, tag.NexusStageCallerInbound), + MetricsHandler: metricsHandler, + Config: serviceConfig, + CallbackTokenGenerator: callbackTokenGenerator, + HistoryClient: historyClient, + // TelemetryInterceptor: telemetryInterceptor, + RequestErrorHandler: requestErrorHandler, + // NamespaceValidationInterceptor: namespaceValidationInterceptor, + // NamespaceRateLimitInterceptor: namespaceRateLimitInterceptor, + // NamespaceConcurrencyLimitInterceptor: namespaceConcurrencyLimitInterceptor, + // RateLimitInterceptor: rateLimitInterceptor, + AuthInterceptor: authInterceptor, + // RedirectionInterceptor: redirectionInterceptor, + ForwardingClients: forwardingClients, + HTTPTraceProvider: httpTraceProvider, + NexusForwarder: nexusForwarder, + nexusInterceptors: nexusInterceptors, + clientVersionChecker: headers.NewDefaultVersionChecker(), + preProcessErrorsCounter: metricsHandler.Counter(metrics.NexusCompletionRequestPreProcessErrors.Name()), } } @@ -179,7 +196,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus rCtx.originalHeaders = r.HTTPRequest.Header.Clone() } ctx = rCtx.augmentContext(ctx, r.HTTPRequest.Header) - defer rCtx.capturePanicAndRecordMetrics(&ctx, &retErr) + defer captureOperationPanic(rCtx.logger, &retErr) if r.HTTPRequest.URL.Path != commonnexus.PathCompletionCallbackNoIdentifier { nsNameEscaped := commonnexus.RouteCompletionCallback.Deserialize(mux.Vars(r.HTTPRequest)) @@ -197,13 +214,37 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid callback token") } } + ctx, err = rCtx.parseTLSAndAuthInfo(ctx, r) + if err != nil { + return err + } - if err := rCtx.interceptRequest(ctx, r); err != nil { - if _, ok := errors.AsType[*serviceerror.NamespaceNotActive](err); ok { - return h.forwardCompleteOperation(ctx, r, rCtx) + interceptorInput := interceptor.NewCompleteNexusOpInput(ns.Name().String(), r) + interceptorInput.WithForwardingInfo(interceptor.NexusForwardingInfo{ + OriginalRequestHeaders: rCtx.originalHeaders, + BusinessID: rCtx.businessID, + }) + finalHandler := func(ctx context.Context, _ interceptor.NexusInterceptorInput) (any, error) { + return nil, h.completeOperationRequest(ctx, logger, completion, r, rCtx) + } + _, err = interceptor.ChainNexusInterceptors(finalHandler, h.nexusInterceptors)(ctx, interceptorInput) + if err != nil { + if taggedErr, ok := errors.AsType[*interceptor.InterceptorError](err); ok { + return taggedErr.Err } return err } + return nil +} + +func (h *nexusCompletionHandler) completeOperationRequest( + ctx context.Context, + logger log.Logger, + completion *tokenspb.NexusOperationCompletion, + r *nexusrpc.CompletionRequest, + rCtx *requestContext, +) error { + ns := rCtx.namespace tokenLimit := h.Config.MaxNexusOperationTokenLength(ns.Name().String()) if len(r.OperationToken) > tokenLimit { return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "operation token length exceeds allowed limit (%d/%d)", len(r.OperationToken), tokenLimit) @@ -232,7 +273,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid completion state") } - err = h.completeOperation(ctx, logger, completion, successPayload, r, links, h.Config.EnableChasm(ns.Name().String())) + err := h.completeOperation(ctx, logger, completion, successPayload, r, links, h.Config.EnableChasm(ns.Name().String())) if err == nil { return nil } @@ -398,68 +439,6 @@ func (h *nexusCompletionHandler) completeChasmOperation( return err } -func (h *nexusCompletionHandler) forwardCompleteOperation(ctx context.Context, r *nexusrpc.CompletionRequest, rCtx *requestContext) error { - targetCluster := rCtx.namespace.ActiveClusterName(namespace.RoutingKey{ID: rCtx.businessID}) - logger := log.With( - rCtx.logger, - tag.SourceCluster(h.ClusterMetadata.GetCurrentClusterName()), - tag.TargetCluster(targetCluster), - ) - - client, err := h.ForwardingClients.Get(targetCluster) - if err != nil { - logger.Error("unable to get HTTP client for forward request", tag.Error(err)) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - - forwardURL, err := url.JoinPath(client.BaseURL(), commonnexus.RouteCompletionCallback.Path(rCtx.namespace.Name().String())) - if err != nil { - logger.Error("failed to construct forwarding request URL", tag.Error(err)) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - - if h.HTTPTraceProvider != nil { - traceLogger := log.With(logger, tag.AttemptStart(time.Now().UTC())) - if trace := h.HTTPTraceProvider.NewForwardingTrace(traceLogger); trace != nil { - ctx = httptrace.WithClientTrace(ctx, trace) - } - } - - var completion nexusrpc.CompleteOperationOptions - switch r.State { - case nexus.OperationStateSucceeded: - completion = nexusrpc.CompleteOperationOptions{ - Result: r.Result.Reader, - OperationToken: r.OperationToken, - StartTime: r.StartTime, - CloseTime: r.CloseTime, - Links: r.Links, - } - case nexus.OperationStateFailed, nexus.OperationStateCanceled: - // For unsuccessful operations, the Nexus framework reads and closes the original request body to deserialize - // the failure, so we must construct a new completion to forward. - completion = nexusrpc.CompleteOperationOptions{ - Error: r.Error, - OperationToken: r.OperationToken, - StartTime: r.StartTime, - CloseTime: r.CloseTime, - Links: r.Links, - } - default: - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation state: %q", r.State) - } - - rCtx.originalHeaders.Set(interceptor.DCRedirectionAPIHeaderName, "true") - rCtx.originalHeaders.Set(interceptor.DCRedirectionSourceCellHeaderName, h.ClusterMetadata.GetCurrentClusterName()) - cc := nexusrpc.NewCompletionHTTPClient(nexusrpc.CompletionHTTPClientOptions{ - HTTPCaller: (&forwardingHTTPHeaderWrapper{ - client: client, - originalRequestHeaders: rCtx.originalHeaders, - }).Do, - }) - return cc.CompleteOperation(ctx, forwardURL, completion) -} - func (h *nexusCompletionHTTPHandler) RegisterRoutes(r *mux.Router) { r.Path("/" + commonnexus.RouteCompletionCallback.Representation()).HandlerFunc(func(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, rpc.MaxNexusAPIRequestBodyBytes) @@ -471,21 +450,6 @@ func (h *nexusCompletionHTTPHandler) RegisterRoutes(r *mux.Router) { }) } -type forwardingHTTPHeaderWrapper struct { - client *common.FrontendHTTPClient - originalRequestHeaders http.Header -} - -func (f *forwardingHTTPHeaderWrapper) Do(req *http.Request) (*http.Response, error) { - // For forwarded requests, copy the original HTTP headers without sanitization. - for k, v := range f.originalRequestHeaders { - if req.Header.Get(k) == "" { - req.Header.Set(k, v[0]) - } - } - return f.client.Do(req) -} - type requestContext struct { *nexusCompletionHandler logger log.Logger @@ -493,21 +457,16 @@ type requestContext struct { metricsHandlerForInterceptors metrics.Handler namespace *namespace.Namespace businessID string - cleanupFunctions []func(error) requestStartTime time.Time outcomeTag metrics.Tag - forwarded bool originalHeaders http.Header } func (c *requestContext) augmentContext(ctx context.Context, header http.Header) context.Context { - ctx = metrics.AddMetricsContext(ctx) - ctx = interceptor.AddTelemetryContext(ctx, c.metricsHandlerForInterceptors) - ctx = interceptor.PopulateCallerInfo( - ctx, - func() string { return c.namespace.Name().String() }, - func() string { return nexusCompletionMethodName }, - ) + ctx = interceptor.WithTelemetryContext(ctx, c) + ctx = interceptor.WithNexusAPIName(ctx, nexusCompletionAPIName) + ctx = interceptor.WithNexusNamespace(ctx, c.namespace) + ctx = interceptor.WithNexusEndpointName(ctx, "") if userAgent := header.Get(headerUserAgent); userAgent != "" { // Preserve original strict behavior: only process if exactly one delimiter present. if strings.Count(userAgent, clientNameVersionDelim) == 1 { @@ -523,52 +482,55 @@ func (c *requestContext) augmentContext(ctx context.Context, header http.Header) } } } - return headers.Propagate(ctx) + return ctx } -func (c *requestContext) capturePanicAndRecordMetrics(ctxPtr *context.Context, errPtr *error) { - recovered := recover() //nolint:revive - if recovered != nil { - err, ok := recovered.(error) - if !ok { - err = fmt.Errorf("panic: %v", recovered) - } - - st := string(debug.Stack()) - c.logger.Error("Panic captured", tag.SysStackTrace(st), tag.Error(err)) - *errPtr = err +func (c *requestContext) MetricsHandler(err error) metrics.Handler { + if c.outcomeTag.Key != "" { + return c.metricsHandler.WithTags(c.outcomeTag) } - if *errPtr == nil { - if c.forwarded { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("request_forwarded")) - } else { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("success")) - } - } else if c.outcomeTag.Key != "" { - c.metricsHandler = c.metricsHandler.WithTags(c.outcomeTag) - } else { - if he, ok := errors.AsType[*nexus.HandlerError](*errPtr); ok { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("error_" + strings.ToLower(string(he.Type)))) - } else { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("error_internal")) - } + if err == nil { + return c.metricsHandler.WithTags(metrics.OutcomeTag("success")) + } + if handlerErr, ok := errors.AsType[*nexus.HandlerError](err); ok { + return c.metricsHandler.WithTags(metrics.OutcomeTag("error_" + strings.ToLower(string(handlerErr.Type)))) } + return c.metricsHandler.WithTags(metrics.OutcomeTag("error_internal")) +} - // Record Nexus-specific metrics - c.metricsHandler.Counter(metrics.NexusCompletionRequests.Name()).Record(1) - c.metricsHandler.Histogram(metrics.NexusCompletionLatencyHistogram.Name(), metrics.Milliseconds).Record(time.Since(c.requestStartTime).Milliseconds()) +func (c *requestContext) MetricsHandlerForInterceptors() metrics.Handler { + return c.metricsHandlerForInterceptors +} - // Record general telemetry metrics - metrics.ServiceRequests.With(c.metricsHandlerForInterceptors).Record(1) - c.TelemetryInterceptor.RecordLatencyMetrics(*ctxPtr, c.requestStartTime, c.metricsHandlerForInterceptors) +func (c *requestContext) MetricsLogger() log.Logger { + return c.logger +} - for _, fn := range c.cleanupFunctions { - fn(*errPtr) - } +func (c *requestContext) SetMetricsOutcome(outcome string) { + c.outcomeTag = metrics.OutcomeTag(outcome) } -// TODO(bergundy): Merge this with the interceptRequest method in nexus_handler.go. -func (c *requestContext) interceptRequest(ctx context.Context, request *nexusrpc.CompletionRequest) error { +// no-op for completion as it doesnt report back via headers +func (c *requestContext) SetFailureSource(string) {} + +func (c *requestContext) HandleRequestError(err error) { + if err == nil { + return + } + c.RequestErrorHandler.HandleError( + // The request is only read to extract workflow log tags, which is keyed off the + // gRPC full method. Nexus has none, so it is never used. + nil, + "", + c.metricsHandlerForInterceptors, + []tag.Tag{tag.Operation(nexusCompletionMethodNameForMetrics), tag.WorkflowNamespace(c.namespace.Name().String())}, + err, + c.namespace.Name(), + ) +} + +// enrich context with authInfo +func (c *requestContext) parseTLSAndAuthInfo(ctx context.Context, request *nexusrpc.CompletionRequest) (context.Context, error) { var tlsInfo *credentials.TLSInfo if request.HTTPRequest.TLS != nil { tlsInfo = &credentials.TLSInfo{ @@ -580,112 +542,12 @@ func (c *requestContext) interceptRequest(ctx context.Context, request *nexusrpc authInfo := c.AuthInterceptor.GetAuthInfo(tlsInfo, request.HTTPRequest.Header, func() string { return "" // TODO: support audience getter }) - - var claims *authorization.Claims - var err error - if authInfo != nil { - claims, err = c.AuthInterceptor.GetClaims(authInfo) - if err != nil { - return err - } - // Make the auth info and claims available on the context. - ctx = c.AuthInterceptor.EnhanceContext(ctx, authInfo, claims) - } - - _, err = c.AuthInterceptor.Authorize(ctx, claims, &authorization.CallTarget{ - APIName: nexusCompletionAPIName, - Namespace: c.namespace.Name().String(), - Request: request, - }) - if err != nil { - // If frontend.exposeAuthorizerErrors is false, Authorize err is either an explicitly set reason, or a generic - // "Request unauthorized." message. - // Otherwise, expose the underlying error. - if permissionDeniedError, ok := errors.AsType[*serviceerror.PermissionDenied](err); ok { - c.outcomeTag = metrics.OutcomeTag("unauthorized") - return commonnexus.AdaptAuthorizeError(permissionDeniedError) - } - c.outcomeTag = metrics.OutcomeTag("internal_auth_error") - c.logger.Error("Authorization internal error with processing nexus callback", tag.Error(err)) - return commonnexus.ConvertGRPCError(err, false) - } - - if err := c.NamespaceValidationInterceptor.ValidateState(c.namespace, nexusCompletionAPIName, c.businessID); err != nil { - c.outcomeTag = metrics.OutcomeTag("invalid_namespace_state") - return commonnexus.ConvertGRPCError(err, false) + if authInfo == nil { + return ctx, nil } - - // Redirect if current cluster is passive for this namespace. - if c.namespace.ActiveClusterName(namespace.RoutingKey{ID: c.businessID}) != c.ClusterMetadata.GetCurrentClusterName() { - if c.shouldForwardRequest(ctx, request.HTTPRequest.Header, c.businessID) { - c.forwarded = true - handler, forwardStartTime := c.RedirectionInterceptor.BeforeCall(nexusCompletionMethodName) - c.cleanupFunctions = append(c.cleanupFunctions, func(retErr error) { - c.RedirectionInterceptor.AfterCall(handler, forwardStartTime, c.namespace.ActiveClusterName(namespace.RoutingKey{ID: c.businessID}), c.namespace.Name().String(), retErr) - }) - // Handler methods should have special logic to forward requests if this method returns a serviceerror.NamespaceNotActive error. - return serviceerror.NewNamespaceNotActive(c.namespace.Name().String(), c.ClusterMetadata.GetCurrentClusterName(), c.namespace.ActiveClusterName(namespace.RoutingKey{ID: c.businessID})) - } - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("namespace_inactive_forwarding_disabled")) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive") - } - - c.cleanupFunctions = append(c.cleanupFunctions, func(retErr error) { - if retErr != nil { - c.RequestErrorHandler.HandleError( - request, - "", - c.metricsHandlerForInterceptors, - []tag.Tag{tag.Operation(nexusCompletionMethodName), tag.WorkflowNamespace(c.namespace.Name().String())}, - retErr, - c.namespace.Name(), - ) - } - }) - - cleanup, err := c.NamespaceConcurrencyLimitInterceptor.Allow(c.namespace.Name(), nexusCompletionAPIName, c.metricsHandlerForInterceptors, request) - c.cleanupFunctions = append(c.cleanupFunctions, func(error) { cleanup() }) - if err != nil { - c.outcomeTag = metrics.OutcomeTag("namespace_concurrency_limited") - return commonnexus.ConvertGRPCError(err, false) - } - - if err := c.NamespaceRateLimitInterceptor.Allow( - ctx, - c.namespace.Name(), - nexusCompletionAPIName, - request.HTTPRequest.Header, - ); err != nil { - c.outcomeTag = metrics.OutcomeTag("namespace_rate_limited") - return commonnexus.ConvertGRPCError(err, true) - } - - if err := c.RateLimitInterceptor.Allow(nexusCompletionAPIName, request.HTTPRequest.Header); err != nil { - c.outcomeTag = metrics.OutcomeTag("global_rate_limited") - return commonnexus.ConvertGRPCError(err, true) - } - - if err := c.clientVersionChecker.ClientSupported(ctx); err != nil { - c.outcomeTag = metrics.OutcomeTag("unsupported_client") - return commonnexus.ConvertGRPCError(err, true) - } - - return nil -} - -// TODO: copied from nexus_handler.go; should be combined with other intercept logic. -// Combines logic from RedirectionInterceptor.redirectionAllowed and some from -// SelectedAPIsForwardingRedirectionPolicy.getTargetClusterAndIsNamespaceNotActiveAutoForwarding so all -// redirection conditions can be checked at once. If either of those methods are updated, this should -// be kept in sync. -func (c *requestContext) shouldForwardRequest(ctx context.Context, header http.Header, businessID string) bool { - redirectHeader := header.Get(interceptor.DCRedirectionContextHeaderName) - redirectAllowed, err := strconv.ParseBool(redirectHeader) + claims, err := c.AuthInterceptor.GetClaims(authInfo) if err != nil { - redirectAllowed = true + return nil, err } - return redirectAllowed && - c.RedirectionInterceptor.RedirectionAllowed(ctx) && - c.namespace.IsGlobalNamespace() && - c.Config.EnableNamespaceNotActiveAutoForwarding(c.namespace.Name().String()) + return c.AuthInterceptor.EnhanceContext(ctx, authInfo, claims), nil } diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index 356014f64f7..108220a743d 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -9,20 +9,17 @@ import ( "net/url" "regexp" "runtime/debug" - "strconv" "strings" "sync" "time" "github.com/nexus-rpc/sdk-go/nexus" - "go.opentelemetry.io/otel/trace" enumspb "go.temporal.io/api/enums/v1" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/server/api/matchingservice/v1" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" - "go.temporal.io/server/common" "go.temporal.io/server/common/authorization" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/dynamicconfig" @@ -85,51 +82,6 @@ type operationContext struct { forwardingEnabledForNamespace dynamicconfig.BoolPropertyFnWithNamespaceFilter headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig] - cleanupFunctions []func(map[string]string, error) -} - -func (c *operationContext) annotateServerSpan( - ctx context.Context, - service, operation, requestID string, -) { - nexusrpc.AnnotateServerSpan(trace.SpanFromContext(ctx), nexusrpc.ServerSpanAttributes{ - Endpoint: c.endpointName, - Service: service, - Operation: operation, - RequestID: requestID, - }) -} - -// Panic handler and metrics recording function. -// Used as a deferred statement in Nexus handler methods. -func (c *operationContext) capturePanicAndRecordMetrics(ctxPtr *context.Context, errPtr *error) { - recovered := recover() //nolint:revive - if recovered != nil { - err, ok := recovered.(error) - if !ok { - err = fmt.Errorf("panic: %v", recovered) - } - - st := string(debug.Stack()) - - c.logger.Error("Panic captured", tag.SysStackTrace(st), tag.Error(err)) - *errPtr = err - } - - // Record Nexus-specific metrics - metrics.NexusRequests.With(c.metricsHandler).Record(1) - metrics.NexusLatency.With(c.metricsHandler).Record(time.Since(c.requestStartTime)) - if *errPtr != nil { - metrics.NexusRequestErrors.With(c.metricsHandler).Record(1) - } - - // Record general telemetry metrics - metrics.ServiceRequests.With(c.metricsHandlerForInterceptors).Record(1) - c.telemetryInterceptor.RecordLatencyMetrics(*ctxPtr, c.requestStartTime, c.metricsHandlerForInterceptors) - - for _, fn := range c.cleanupFunctions { - fn(c.responseHeaders, *errPtr) - } } func (c *operationContext) matchingRequest(req *nexuspb.Request) *matchingservice.DispatchNexusTaskRequest { @@ -142,13 +94,10 @@ func (c *operationContext) matchingRequest(req *nexuspb.Request) *matchingservic } func (c *operationContext) augmentContext(ctx context.Context, header nexus.Header) context.Context { - ctx = metrics.AddMetricsContext(ctx) - ctx = interceptor.AddTelemetryContext(ctx, c.metricsHandlerForInterceptors) - ctx = interceptor.PopulateCallerInfo( - ctx, - func() string { return c.namespaceName }, - func() string { return c.method }, - ) + ctx = interceptor.WithTelemetryContext(ctx, c) + ctx = interceptor.WithNexusAPIName(ctx, c.apiName) + ctx = interceptor.WithNexusEndpointName(ctx, c.endpointName) + ctx = interceptor.WithNexusNamespace(ctx, c.namespace) if userAgent, ok := header[headerUserAgent]; ok { // Use SplitN for efficiency but enforce exactly one delimiter to preserve the // original (pre-SplitN) strictness where additional delimiters cause us to ignore @@ -166,140 +115,81 @@ func (c *operationContext) augmentContext(ctx context.Context, header nexus.Head } } } - return headers.Propagate(ctx) + return ctx } -func (c *operationContext) interceptRequest( - ctx context.Context, - request *matchingservice.DispatchNexusTaskRequest, - header nexus.Header, -) error { - _, err := c.auth.Authorize(ctx, c.claims, &authorization.CallTarget{ - APIName: c.apiName, - Namespace: c.namespaceName, - NexusEndpointName: c.endpointName, - Request: request, - }) - if err != nil { - // If frontend.exposeAuthorizerErrors is false, Authorize err is either an explicitly set reason, or a generic - // "Request unauthorized." message. - // Otherwise, expose the underlying error. - if permissionDeniedError, ok := errors.AsType[*serviceerror.PermissionDenied](err); ok { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("unauthorized")) - return commonnexus.AdaptAuthorizeError(permissionDeniedError) - } - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("internal_auth_error")) - c.logger.Error("Authorization internal error with processing nexus request", tag.Error(err)) - return commonnexus.ConvertGRPCError(err, false) - } +func (c *operationContext) MetricsHandler(_ error) metrics.Handler { + // start/cancel handlers already have the error, just return the handler + return c.metricsHandler +} - // Nexus requests are not tied to a business ID, hence the empty string. - if err := c.namespaceValidationInterceptor.ValidateState(c.namespace, c.apiName, namespace.EmptyBusinessID); err != nil { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("invalid_namespace_state")) - return commonnexus.ConvertGRPCError(err, false) - } +func (c *operationContext) MetricsHandlerForInterceptors() metrics.Handler { + return c.metricsHandlerForInterceptors +} - //nolint:forbidigo // Nexus requests are not tied to a business ID by design (see line 184) - if !c.namespace.ActiveInCluster(c.clusterMetadata.GetCurrentClusterName()) { - if c.shouldForwardRequest(ctx, header) { - // Handler methods should have special logic to forward requests if this method returns - // a serviceerror.NamespaceNotActive error. - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("request_forwarded")) - handler, forwardStartTime := c.redirectionInterceptor.BeforeCall(c.apiName) - c.cleanupFunctions = append(c.cleanupFunctions, func(_ map[string]string, retErr error) { - c.redirectionInterceptor.AfterCall(handler, forwardStartTime, c.namespace.ActiveClusterName(namespace.RoutingKey{}), c.namespace.Name().String(), retErr) - }) - return serviceerror.NewNamespaceNotActive( - c.namespaceName, - c.clusterMetadata.GetCurrentClusterName(), - c.namespace.ActiveClusterName(namespace.RoutingKey{}), - ) - } - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("namespace_inactive_forwarding_disabled")) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive") - } - - c.cleanupFunctions = append(c.cleanupFunctions, func(respHeaders map[string]string, retErr error) { - if retErr != nil { - if source, ok := respHeaders[commonnexus.FailureSourceHeaderName]; ok && source != commonnexus.FailureSourceWorker { - c.requestErrorHandler.HandleError( - request, - "", - c.metricsHandlerForInterceptors, - []tag.Tag{tag.Operation(c.method), tag.WorkflowNamespace(c.namespaceName)}, - retErr, - c.namespace.Name(), - ) - } - } - }) +func (c *operationContext) MetricsLogger() log.Logger { + return c.logger +} - cleanup, err := c.namespaceConcurrencyLimitInterceptor.Allow( - c.namespace.Name(), - c.apiName, +func (c *operationContext) SetMetricsOutcome(outcome string) { + c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag(outcome)) +} + +func (c *operationContext) SetFailureSource(source string) { + c.setFailureSource(source) +} + +// replaces registerRequestErrorHandler and the cleanupFunctions +// registry. Errors that a worker produced are already reported by the worker's own +// cluster, so only other sources are handled here. +func (c *operationContext) HandleRequestError(err error) { + if err == nil { + return + } + source, ok := c.responseHeaders[commonnexus.FailureSourceHeaderName] + if !ok || source == commonnexus.FailureSourceWorker { + return + } + c.requestErrorHandler.HandleError( + // The request is only read to extract workflow log tags, which is keyed off the + // gRPC full method. Nexus has none, so it is never used. + nil, + "", c.metricsHandlerForInterceptors, - request, + []tag.Tag{tag.Operation(c.method), tag.WorkflowNamespace(c.namespaceName)}, + err, + c.namespace.Name(), ) - c.cleanupFunctions = append(c.cleanupFunctions, func(map[string]string, error) { cleanup() }) - if err != nil { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("namespace_concurrency_limited")) - return commonnexus.ConvertGRPCError(err, false) - } +} - if err := c.namespaceRateLimitInterceptor.Allow( - ctx, - c.namespace.Name(), - c.apiName, - header, - ); err != nil { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("namespace_rate_limited")) - return commonnexus.ConvertGRPCError(err, true) +// required as operations might panic before the interceptor chain is +// invoked and panics are handled by the outermost telemetry interceptor +func captureOperationPanic(logger log.Logger, errPtr *error) { + recovered := recover() //nolint:revive + if recovered == nil { + return } - - if err := c.rateLimitInterceptor.Allow(c.apiName, header); err != nil { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("global_rate_limited")) - return commonnexus.ConvertGRPCError(err, true) + err, ok := recovered.(error) + if !ok { + err = fmt.Errorf("panic: %v", recovered) } + logger.Error("Panic captured", tag.SysStackTrace(string(debug.Stack())), tag.Error(err)) + *errPtr = err +} - if err := c.clientVersionChecker.ClientSupported(ctx); err != nil { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("unsupported_client")) - converted := commonnexus.ConvertGRPCError(err, true) - return converted +func (c *operationContext) sanitizeRequestHeaders(request *matchingservice.DispatchNexusTaskRequest) { + if request.GetRequest().GetHeader() == nil { + return } - // THIS MUST BE THE LAST STEP IN interceptRequest. - // Sanitize headers. - if request.GetRequest().GetHeader() != nil { - // Making a copy to ensure the original map is not modified as it might be used somewhere else. - sanitizedHeaders := make(map[string]string, len(request.Request.Header)) - headersBlacklist := c.headersBlacklist() - for name, value := range request.Request.Header { - if !headersBlacklist.MatchString(name) { - sanitizedHeaders[name] = value - } + sanitizedHeaders := make(map[string]string, len(request.Request.Header)) + headersBlacklist := c.headersBlacklist() + for name, value := range request.Request.Header { + if !headersBlacklist.MatchString(name) { + sanitizedHeaders[name] = value } - request.Request.Header = sanitizedHeaders } - - // DO NOT ADD ANY STEPS HERE. ALL STEPS MUST BE BEFORE HEADERS SANITIZATION. - - return nil -} - -// Combines logic from RedirectionInterceptor.redirectionAllowed and some from -// SelectedAPIsForwardingRedirectionPolicy.getTargetClusterAndIsNamespaceNotActiveAutoForwarding so all -// redirection conditions can be checked at once. If either of those methods are updated, this should -// be kept in sync. -func (c *operationContext) shouldForwardRequest(ctx context.Context, header nexus.Header) bool { - redirectHeader := header.Get(interceptor.DCRedirectionContextHeaderName) - redirectAllowed, err := strconv.ParseBool(redirectHeader) - if err != nil { - redirectAllowed = true - } - return redirectAllowed && - c.redirectionInterceptor.RedirectionAllowed(ctx) && - c.namespace.IsGlobalNamespace() && - c.forwardingEnabledForNamespace(c.namespaceName) + request.Request.Header = sanitizedHeaders } // enrichNexusOperationMetrics enhances metrics with additional Nexus operation context based on configuration. @@ -345,15 +235,15 @@ type nexusContextKey struct{} // Dispatches Nexus requests as Nexus tasks to workers via matching. type nexusHandler struct { nexus.UnimplementedHandler - logger log.Logger - metricsHandler metrics.Handler - clusterMetadata cluster.Metadata - namespaceRegistry namespace.Registry - matchingClient matchingservice.MatchingServiceClient - auth *authorization.Interceptor - telemetryInterceptor *interceptor.TelemetryInterceptor - requestErrorHandler *interceptor.RequestErrorHandler - redirectionInterceptor *interceptor.Redirection + logger log.Logger + metricsHandler metrics.Handler + clusterMetadata cluster.Metadata + namespaceRegistry namespace.Registry + matchingClient matchingservice.MatchingServiceClient + auth *authorization.Interceptor + // telemetryInterceptor *interceptor.TelemetryInterceptor + // requestErrorHandler *interceptor.RequestErrorHandler + // redirectionInterceptor *interceptor.Redirection forwardingEnabledForNamespace dynamicconfig.BoolPropertyFnWithNamespaceFilter forwardingClients *cluster.FrontendHTTPClientCache payloadSizeLimit dynamicconfig.IntPropertyFnWithNamespaceFilter @@ -361,6 +251,7 @@ type nexusHandler struct { useForwardByEndpoint dynamicconfig.BoolPropertyFn metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig] httpTraceProvider commonnexus.HTTPClientTraceProvider + nexusInterceptors []interceptor.NexusInterceptor } // Extracts a nexusContext from the given ctx and returns an operationContext with tagged metrics and logging. @@ -371,18 +262,17 @@ func (h *nexusHandler) getOperationContext(ctx context.Context, method string) ( return nil, errors.New("no nexus context set on context") } oc := operationContext{ - nexusContext: nc, - method: method, - clusterMetadata: h.clusterMetadata, - clientVersionChecker: headers.NewDefaultVersionChecker(), - auth: h.auth, - telemetryInterceptor: h.telemetryInterceptor, - requestErrorHandler: h.requestErrorHandler, - redirectionInterceptor: h.redirectionInterceptor, + nexusContext: nc, + method: method, + clusterMetadata: h.clusterMetadata, + clientVersionChecker: headers.NewDefaultVersionChecker(), + auth: h.auth, + // telemetryInterceptor: h.telemetryInterceptor, + // requestErrorHandler: h.requestErrorHandler, + // redirectionInterceptor: h.redirectionInterceptor, forwardingEnabledForNamespace: h.forwardingEnabledForNamespace, headersBlacklist: h.headersBlacklist, metricTagConfig: h.metricTagConfig, - cleanupFunctions: make([]func(map[string]string, error), 0), } oc.metricsHandlerForInterceptors = h.metricsHandler.WithTags( metrics.OperationTag(method), @@ -427,8 +317,7 @@ func (h *nexusHandler) StartOperation( ctx = oc.augmentContext(ctx, options.Header) oc.enrichNexusOperationMetrics(service, operation, options.Header) oc.enrichNexusOperationLogs(service, operation, options.RequestID) - oc.annotateServerSpan(ctx, service, operation, options.RequestID) - defer oc.capturePanicAndRecordMetrics(&ctx, &retErr) + defer captureOperationPanic(oc.logger, &retErr) var links []*nexuspb.Link for _, nexusLink := range options.Links { @@ -457,13 +346,42 @@ func (h *nexusHandler) StartOperation( }, }) - if err := oc.interceptRequest(ctx, request, options.Header); err != nil { - if _, ok := errors.AsType[*serviceerror.NamespaceNotActive](err); ok { - return h.forwardStartOperation(ctx, service, operation, input, options, oc) + finalHandler := func(ctx context.Context, _ interceptor.NexusInterceptorInput) (any, error) { + return h.finalStartHandler(ctx, oc, operation, input, &startOperationRequest, request) + } + + nexusOpInput := interceptor.NewStartNexusOpInput(service, operation, oc.namespaceName, options, input) + nexusOpInput.WithForwardingInfo(interceptor.NexusForwardingInfo{ + OriginalRequestHeaders: oc.originalRequestHeaders, + TaskQueue: oc.taskQueue, + EndpointID: oc.endpointID, + EndpointName: oc.endpointName, + }) + chainedHandler := interceptor.ChainNexusInterceptors(finalHandler, h.nexusInterceptors) + out, err := chainedHandler(ctx, nexusOpInput) + if err != nil { + if taggedErr, ok := errors.AsType[*interceptor.InterceptorError](err); ok { + return nil, taggedErr.Err } return nil, err } + res, ok := out.(nexus.HandlerStartOperationResult[any]) + if !ok { + return nil, fmt.Errorf("unexpected Nexus start interceptor result type %T", out) + } + return res, nil +} +//nolint:revive,cognitive-complexity: this is just a shift of existing code +func (h *nexusHandler) finalStartHandler(ctx context.Context, + oc *operationContext, + operation string, + input *nexus.LazyValue, + startOperationRequest *nexuspb.StartOperationRequest, + request *matchingservice.DispatchNexusTaskRequest, +) (any, error) { + oc.sanitizeRequestHeaders(request) + var err error // Transform nexus Content to temporal Payload with common/nexus PayloadSerializer. if err = input.Consume(&startOperationRequest.Payload); err != nil { oc.logger.Warn("invalid input", tag.Error(err)) @@ -510,54 +428,6 @@ func parseLinks(links []*nexuspb.Link, logger log.Logger) []nexus.Link { return nexusLinks } -// forwardStartOperation forwards the StartOperation request to the active cluster using an HTTP request. -// Inputs and response values are passed as Reader objects to avoid reading bodies and bypass serialization. -func (h *nexusHandler) forwardStartOperation( - ctx context.Context, - service string, - operation string, - input *nexus.LazyValue, - options nexus.StartOperationOptions, - oc *operationContext, -) (nexus.HandlerStartOperationResult[any], error) { - options.Header[interceptor.DCRedirectionAPIHeaderName] = "true" - options.Header[interceptor.DCRedirectionSourceCellHeaderName] = h.clusterMetadata.GetCurrentClusterName() - - client, err := h.nexusClientForActiveCluster(oc, service) - if err != nil { - return nil, err - } - - if h.httpTraceProvider != nil { - traceLogger := log.With(h.logger, - tag.Operation(oc.method), - tag.WorkflowNamespace(oc.namespaceName), - tag.RequestID(options.RequestID), - tag.NexusOperation(operation), - tag.Endpoint(oc.endpointName), - tag.AttemptStart(time.Now().UTC()), - tag.SourceCluster(h.clusterMetadata.GetCurrentClusterName()), - tag.TargetCluster(oc.namespace.ActiveClusterName(namespace.RoutingKey{})), - ) - if trace := h.httpTraceProvider.NewForwardingTrace(traceLogger); trace != nil { - ctx = httptrace.WithClientTrace(ctx, trace) - } - } - - resp, err := client.StartOperation(ctx, operation, input.Reader, options) - if err != nil { - oc.logger.Error("received error from remote cluster for forwarded Nexus start operation request.", tag.Error(err)) - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("forwarded_request_error")) - return nil, err - } - - if resp.Successful != nil { - return &nexus.HandlerStartOperationResultSync[any]{Value: resp.Successful.Reader}, nil - } - // If Nexus client did not return an error, one of Successful or Pending will always be set. - return &nexus.HandlerStartOperationResultAsync{OperationToken: resp.Pending.Token}, nil -} - func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, token string, options nexus.CancelOperationOptions) (retErr error) { oc, err := h.getOperationContext(ctx, "CancelNexusOperation") if err != nil { @@ -566,8 +436,7 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, ctx = oc.augmentContext(ctx, options.Header) oc.enrichNexusOperationMetrics(service, operation, options.Header) oc.enrichNexusOperationLogs(service, operation, "") - oc.annotateServerSpan(ctx, service, operation, "") - defer oc.capturePanicAndRecordMetrics(&ctx, &retErr) + defer captureOperationPanic(oc.logger, &retErr) request := oc.matchingRequest(&nexuspb.Request{ Header: options.Header, @@ -585,12 +454,36 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, TemporalFailureResponses: oc.callerFailureSupport, }, }) - if err := oc.interceptRequest(ctx, request, options.Header); err != nil { - if _, ok := errors.AsType[*serviceerror.NamespaceNotActive](err); ok { - return h.forwardCancelOperation(ctx, service, operation, token, options, oc) + + finalHandler := func(ctx context.Context, _ interceptor.NexusInterceptorInput) (any, error) { + return nil, h.finalCancelHandler(ctx, oc, operation, request) + } + + nexusInterceptorInput := interceptor.NewCancelNexusOpInput(service, operation, oc.namespaceName, options, token) + nexusInterceptorInput.WithForwardingInfo(interceptor.NexusForwardingInfo{ + OriginalRequestHeaders: oc.originalRequestHeaders, + TaskQueue: oc.taskQueue, + EndpointID: oc.endpointID, + EndpointName: oc.endpointName, + }) + chainedHandler := interceptor.ChainNexusInterceptors(finalHandler, h.nexusInterceptors) + _, err = chainedHandler(ctx, nexusInterceptorInput) + if err != nil { + if taggedErr, ok := errors.AsType[*interceptor.InterceptorError](err); ok { + return taggedErr.Err } return err } + return nil +} + +func (h *nexusHandler) finalCancelHandler( + ctx context.Context, + oc *operationContext, + operation string, + request *matchingservice.DispatchNexusTaskRequest, +) error { + oc.sanitizeRequestHeaders(request) // Dispatch the request to be sync matched with a worker polling on the nexusContext taskQueue. // matchingClient sets a context timeout of 60 seconds for this request, this should be enough for any Nexus @@ -699,34 +592,27 @@ func (h *nexusHandler) nexusClientForActiveCluster(oc *operationContext, service }) } +func convertOutcomeToNexusHandlerError(resp *matchingservice.DispatchNexusTaskResponse_HandlerError) *nexus.HandlerError { + var retryBehavior nexus.HandlerErrorRetryBehavior + // nolint:exhaustive // unspecified is the default + switch resp.HandlerError.RetryBehavior { + case enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE: + retryBehavior = nexus.HandlerErrorRetryBehaviorRetryable + case enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_NON_RETRYABLE: + retryBehavior = nexus.HandlerErrorRetryBehaviorNonRetryable + } + // nolint:staticcheck // Deprecated function still in use for backward compatibility. + cause := commonnexus.ProtoFailureToNexusFailure(resp.HandlerError.GetFailure()) + return &nexus.HandlerError{ + // nolint:staticcheck // Deprecated function still in use for backward compatibility. + Type: nexus.HandlerErrorType(resp.HandlerError.GetErrorType()), + RetryBehavior: retryBehavior, + Cause: &nexus.FailureError{Failure: cause}, + } +} + func (nc *nexusContext) setFailureSource(source string) { nc.responseHeadersMutex.Lock() defer nc.responseHeadersMutex.Unlock() nc.responseHeaders[commonnexus.FailureSourceHeaderName] = source } - -type forwardingHttpHeaderWrapper struct { - client *common.FrontendHTTPClient - nc *nexusContext - originalRequestHeaders http.Header -} - -func (f *forwardingHttpHeaderWrapper) Do(req *http.Request) (*http.Response, error) { - // For forwarded requests, copy the original HTTP headers without sanitization. - for k, v := range f.originalRequestHeaders { - if req.Header.Get(k) == "" { - req.Header.Set(k, v[0]) - } - } - - response, err := f.client.Do(req) - if err != nil { - return nil, err - } - - if failureSource := response.Header.Get(commonnexus.FailureSourceHeaderName); failureSource != "" { - f.nc.setFailureSource(failureSource) - } - - return response, nil -} diff --git a/service/frontend/nexus_handler_test.go b/service/frontend/nexus_handler_test.go index 17891daea3f..7f2cbdfbf28 100644 --- a/service/frontend/nexus_handler_test.go +++ b/service/frontend/nexus_handler_test.go @@ -3,16 +3,10 @@ package frontend import ( "context" "errors" - "testing" "time" "github.com/google/uuid" - "github.com/nexus-rpc/sdk-go/nexus" - "github.com/stretchr/testify/require" enumspb "go.temporal.io/api/enums/v1" - nexuspb "go.temporal.io/api/nexus/v1" - "go.temporal.io/api/serviceerror" - "go.temporal.io/server/api/matchingservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/authorization" "go.temporal.io/server/common/clock" @@ -28,7 +22,6 @@ import ( "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/quotas" "go.temporal.io/server/common/rpc/interceptor" - "go.temporal.io/server/common/util" ) type mockAuthorizer struct{} @@ -179,204 +172,3 @@ func newOperationContext(options contextOptions) *operationContext { return oc } - -func TestNexusInterceptRequest_InvalidNamespaceState_ResultsInBadRequest(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - var err error - oc := newOperationContext(contextOptions{ - namespaceState: enumspb.NAMESPACE_STATE_DELETED, - quota: 1, - namespaceRateLimitAllow: true, - rateLimitAllow: true, - }) - err = oc.interceptRequest(ctx, &matchingservice.DispatchNexusTaskRequest{}, nexus.Header{}) - var handlerError *nexus.HandlerError - require.ErrorAs(t, err, &handlerError) - require.Equal(t, nexus.HandlerErrorTypeBadRequest, handlerError.Type) - require.Equal(t, "bad request", handlerError.Message) - mh := oc.metricsHandler.(*metricstest.CaptureHandler) //nolint:revive - capture := mh.StartCapture() - oc.metricsHandler.Counter("test").Record(1) - mh.StopCapture(capture) - snap := capture.Snapshot() - require.Len(t, snap["test"], 1) - require.Equal(t, map[string]string{"outcome": "invalid_namespace_state"}, snap["test"][0].Tags) -} - -func TestNexusInterceptRequest_NamespaceConcurrencyLimited_ResultsInResourceExhausted(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - var err error - oc := newOperationContext(contextOptions{ - namespaceState: enumspb.NAMESPACE_STATE_REGISTERED, - quota: 0, - namespaceRateLimitAllow: true, - rateLimitAllow: true, - }) - err = oc.interceptRequest(ctx, &matchingservice.DispatchNexusTaskRequest{}, nexus.Header{}) - var handlerError *nexus.HandlerError - require.ErrorAs(t, err, &handlerError) - require.Equal(t, nexus.HandlerErrorTypeResourceExhausted, handlerError.Type) - require.Equal(t, "resource exhausted", handlerError.Message) - mh := oc.metricsHandler.(*metricstest.CaptureHandler) //nolint:revive - capture := mh.StartCapture() - oc.metricsHandler.Counter("test").Record(1) - mh.StopCapture(capture) - snap := capture.Snapshot() - require.Len(t, snap["test"], 1) - require.Equal(t, map[string]string{"outcome": "namespace_concurrency_limited"}, snap["test"][0].Tags) -} - -func TestNexusInterceptRequest_NamespaceRateLimited_ResultsInResourceExhausted(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - var err error - oc := newOperationContext(contextOptions{ - namespaceState: enumspb.NAMESPACE_STATE_REGISTERED, - quota: 1, - namespaceRateLimitAllow: false, - rateLimitAllow: true, - }) - err = oc.interceptRequest(ctx, &matchingservice.DispatchNexusTaskRequest{}, nexus.Header{}) - var handlerError *nexus.HandlerError - require.ErrorAs(t, err, &handlerError) - require.Equal(t, nexus.HandlerErrorTypeResourceExhausted, handlerError.Type) - require.Equal(t, "namespace rate limit exceeded", handlerError.Message) - mh := oc.metricsHandler.(*metricstest.CaptureHandler) //nolint:revive - capture := mh.StartCapture() - oc.metricsHandler.Counter("test").Record(1) - mh.StopCapture(capture) - snap := capture.Snapshot() - require.Len(t, snap["test"], 1) - require.Equal(t, map[string]string{"outcome": "namespace_rate_limited"}, snap["test"][0].Tags) -} - -func TestNexusInterceptRequest_GlobalRateLimited_ResultsInResourceExhausted(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - var err error - oc := newOperationContext(contextOptions{ - namespaceState: enumspb.NAMESPACE_STATE_REGISTERED, - quota: 1, - namespaceRateLimitAllow: true, - rateLimitAllow: false, - }) - err = oc.interceptRequest(ctx, &matchingservice.DispatchNexusTaskRequest{}, nexus.Header{}) - var handlerError *nexus.HandlerError - require.ErrorAs(t, err, &handlerError) - require.Equal(t, nexus.HandlerErrorTypeResourceExhausted, handlerError.Type) - require.Equal(t, "service rate limit exceeded", handlerError.Message) - mh := oc.metricsHandler.(*metricstest.CaptureHandler) //nolint:revive - capture := mh.StartCapture() - oc.metricsHandler.Counter("test").Record(1) - mh.StopCapture(capture) - snap := capture.Snapshot() - require.Len(t, snap["test"], 1) - require.Equal(t, map[string]string{"outcome": "global_rate_limited"}, snap["test"][0].Tags) -} - -func TestNexusInterceptRequest_ForwardingDisabled_ResultsInUnavailable(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - var err error - oc := newOperationContext(contextOptions{ - namespaceState: enumspb.NAMESPACE_STATE_REGISTERED, - namespacePassive: true, - quota: 1, - namespaceRateLimitAllow: true, - rateLimitAllow: true, - redirectAllow: false, - }) - err = oc.interceptRequest(ctx, &matchingservice.DispatchNexusTaskRequest{}, nexus.Header{}) - var handlerError *nexus.HandlerError - require.ErrorAs(t, err, &handlerError) - require.Equal(t, nexus.HandlerErrorTypeUnavailable, handlerError.Type) - mh := oc.metricsHandler.(*metricstest.CaptureHandler) //nolint:revive - capture := mh.StartCapture() - oc.metricsHandler.Counter("test").Record(1) - mh.StopCapture(capture) - snap := capture.Snapshot() - require.Len(t, snap["test"], 1) - require.Equal(t, map[string]string{"outcome": "namespace_inactive_forwarding_disabled"}, snap["test"][0].Tags) -} - -func TestNexusInterceptRequest_ForwardingEnabled_ResultsInNotActiveError(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - var err error - oc := newOperationContext(contextOptions{ - namespaceState: enumspb.NAMESPACE_STATE_REGISTERED, - namespacePassive: true, - quota: 1, - namespaceRateLimitAllow: true, - rateLimitAllow: true, - redirectAllow: true, - }) - err = oc.interceptRequest(ctx, &matchingservice.DispatchNexusTaskRequest{}, nexus.Header{}) - var notActiveErr *serviceerror.NamespaceNotActive - require.ErrorAs(t, err, ¬ActiveErr) - mh := oc.metricsHandler.(*metricstest.CaptureHandler) //nolint:revive - capture := mh.StartCapture() - oc.metricsHandler.Counter("test").Record(1) - mh.StopCapture(capture) - snap := capture.Snapshot() - require.Len(t, snap["test"], 1) - require.Equal(t, map[string]string{"outcome": "request_forwarded"}, snap["test"][0].Tags) -} - -func TestNexusInterceptRequest_InvalidSDKVersion_ResultsInBadRequest(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - var err error - oc := newOperationContext(contextOptions{ - namespaceState: enumspb.NAMESPACE_STATE_REGISTERED, - namespacePassive: false, - quota: 1, - namespaceRateLimitAllow: true, - rateLimitAllow: true, - redirectAllow: true, - }) - header := nexus.Header{headerUserAgent: "Nexus-go-sdk/v99.0.0"} - ctx = oc.augmentContext(ctx, header) - err = oc.interceptRequest(ctx, &matchingservice.DispatchNexusTaskRequest{}, header) - var handlerError *nexus.HandlerError - require.ErrorAs(t, err, &handlerError) - require.Equal(t, nexus.HandlerErrorTypeBadRequest, handlerError.Type) - mh := oc.metricsHandler.(*metricstest.CaptureHandler) //nolint:revive - capture := mh.StartCapture() - oc.metricsHandler.Counter("test").Record(1) - mh.StopCapture(capture) - snap := capture.Snapshot() - require.Len(t, snap["test"], 1) - require.Equal(t, map[string]string{"outcome": "unsupported_client"}, snap["test"][0].Tags) -} - -func TestNexusInterceptRequest_HeadersSanitization(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - var err error - oc := newOperationContext(contextOptions{ - namespaceState: enumspb.NAMESPACE_STATE_REGISTERED, - namespacePassive: false, - quota: 1, - namespaceRateLimitAllow: true, - rateLimitAllow: true, - headersBlacklist: []string{"delete-*", "remove-*"}, - }) - initialHeader := nexus.Header{ - "ok-header": "ok", - "delete-foo": "foo", - "delete-bar": "bar", - "remove-zzz": "zzz", - } - header := util.CloneMapNonNil(initialHeader) - ctx = oc.augmentContext(ctx, header) - request := &matchingservice.DispatchNexusTaskRequest{ - Request: &nexuspb.Request{Header: header}, - } - err = oc.interceptRequest(ctx, request, header) - require.NoError(t, err) - require.Equal(t, initialHeader, header) - require.Equal(t, map[string]string{"ok-header": "ok"}, request.Request.Header) -} diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index 93998d2f17e..7e0126ae31b 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -66,11 +66,34 @@ func NewNexusOperationHTTPHandler( namespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor, namespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor, rateLimitInterceptor *interceptor.RateLimitInterceptor, + sdkVersionInterceptor *interceptor.SDKVersionInterceptor, + callerInfoInterceptor *interceptor.CallerInfoInterceptor, + nexusForwarder *nexusForwardingInterceptor, + customNexusInterceptors []interceptor.NexusInterceptor, logger log.Logger, httpTraceProvider commonnexus.HTTPClientTraceProvider, httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter, ) *NexusOperationHTTPHandler { logger = log.With(logger, tag.NexusStageHandlerInbound) + + // draft-review: should we also just make an interceptors provider fx + // so it can be shared/declared in a single place. Maybe not worth it as + // eventual goal is to remove the completion handler and move that into the + // http handler as well + + nexusInterceptors := []interceptor.NexusInterceptor{ + telemetryInterceptor.InterceptNexus, + authInterceptor.InterceptNexus, + nexusForwarder.InterceptNexus, + namespaceValidationInterceptor.InterceptNexus, + namespaceConcurrencyLimitInterceptor.InterceptNexus, + namespaceRateLimitInterceptor.InterceptNexus, + rateLimitInterceptor.InterceptNexus, + sdkVersionInterceptor.InterceptNexus, + callerInfoInterceptor.InterceptNexus, + } + nexusInterceptors = append(nexusInterceptors, customNexusInterceptors...) + return &NexusOperationHTTPHandler{ base: nexusrpc.BaseHTTPHandler{ Logger: log.NewSlogLogger(logger), @@ -88,15 +111,15 @@ func NewNexusOperationHTTPHandler( httpServerHandlerInstrumenter: httpServerHandlerInstrumenter, nexusHandler: nexusrpc.NewHTTPHandler(nexusrpc.HandlerOptions{ Handler: &nexusHandler{ - logger: logger, - metricsHandler: metricsHandler, - clusterMetadata: clusterMetadata, - namespaceRegistry: namespaceRegistry, - matchingClient: matchingservice.MatchingServiceClient(matchingClient), - auth: authInterceptor, - telemetryInterceptor: telemetryInterceptor, - requestErrorHandler: requestErrorHandler, - redirectionInterceptor: redirectionInterceptor, + logger: logger, + metricsHandler: metricsHandler, + clusterMetadata: clusterMetadata, + namespaceRegistry: namespaceRegistry, + matchingClient: matchingservice.MatchingServiceClient(matchingClient), + auth: authInterceptor, + // telemetryInterceptor: telemetryInterceptor, + // requestErrorHandler: requestErrorHandler, + // redirectionInterceptor: redirectionInterceptor, forwardingEnabledForNamespace: serviceConfig.EnableNamespaceNotActiveAutoForwarding, forwardingClients: clientCache, payloadSizeLimit: serviceConfig.BlobSizeLimitError, @@ -104,6 +127,7 @@ func NewNexusOperationHTTPHandler( useForwardByEndpoint: serviceConfig.NexusForwardRequestUseEndpoint, metricTagConfig: serviceConfig.NexusOperationsMetricTagConfig, httpTraceProvider: httpTraceProvider, + nexusInterceptors: nexusInterceptors, }, GetResultTimeout: serviceConfig.KeepAliveMaxConnectionIdle(), Logger: log.NewSlogLogger(logger), diff --git a/temporal/fx.go b/temporal/fx.go index 221b731335a..9471132aa0d 100644 --- a/temporal/fx.go +++ b/temporal/fx.go @@ -47,6 +47,7 @@ import ( "go.temporal.io/server/common/resource" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" + rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" "go.temporal.io/server/common/telemetry" @@ -121,6 +122,8 @@ type ( TokenProvider auth.TokenProvider ServiceHosts map[primitives.ServiceName]static.Hosts + CustomFrontendNexusInterceptors []rpcinterceptor.NexusInterceptor + // below are things that could be over write by server options or may have default if not supplied by serverOptions. Logger log.Logger ClientFactoryProvider client.FactoryProvider @@ -323,6 +326,7 @@ func ServerOptionsProvider(opts []ServerOption) (serverOptionsProvider, error) { CustomVisibilityStore: so.customVisibilityStoreFactory, CustomHistoryArchiverFactory: so.customHistoryArchiverFactory, CustomVisibilityArchiverFactory: so.customVisibilityArchiverFactory, + CustomFrontendNexusInterceptors: so.customFrontendNexusInterceptors, SearchAttributesMapper: so.searchAttributesMapper, CustomFrontendInterceptors: so.customFrontendInterceptors, @@ -397,6 +401,7 @@ type ( PersistenceFactoryProvider persistenceClient.FactoryProviderFn SearchAttributesMapper searchattribute.Mapper CustomFrontendInterceptors []grpc.UnaryServerInterceptor + CustomFrontendNexusInterceptors []rpcinterceptor.NexusInterceptor AdditionalStreamInterceptors []grpc.StreamServerInterceptor Authorizer authorization.Authorizer ClaimMapper authorization.ClaimMapper @@ -594,6 +599,7 @@ func genericFrontendServiceProvider( app := fx.New( params.GetCommonServiceOptions(serviceName), fx.Supply(params.CustomFrontendInterceptors), + fx.Supply(params.CustomFrontendNexusInterceptors), fx.Decorate(func() authorization.ClaimMapper { switch serviceName { case primitives.FrontendService: diff --git a/temporal/server_option.go b/temporal/server_option.go index b72cfe81e8e..fd1cbd41782 100644 --- a/temporal/server_option.go +++ b/temporal/server_option.go @@ -18,6 +18,7 @@ import ( "go.temporal.io/server/common/resolver" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" + rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/testing/testhooks" "google.golang.org/grpc" @@ -209,6 +210,17 @@ func WithChainedFrontendGrpcInterceptors( }) } +// WithChainedFrontendNexusInterceptors sets an orderered chain of custom Nexus interceptors that will be invoked for +// Frontend Nexus API calls. The custom interceptors will be appended to the end of the internal ServerInterceptors +// and invoked in the order that they appear in the supplied list. +func WithChainedFrontendNexusInterceptors( + interceptors ...rpcinterceptor.NexusInterceptor, +) ServerOption { + return applyFunc(func(s *serverOptions) { + s.customFrontendNexusInterceptors = interceptors + }) +} + // WithAdditionalStreamInterceptors sets a chain of ordered custom grpc stream interceptors that will be invoked for all // service gRPC stream calls. The list of custom interceptors will be appended to the end of the internal // ServerInterceptors. The custom interceptors will be invoked in the order as they appear in the supplied list, after diff --git a/temporal/server_options.go b/temporal/server_options.go index 388425738cb..ff1086e7a94 100644 --- a/temporal/server_options.go +++ b/temporal/server_options.go @@ -21,6 +21,7 @@ import ( "go.temporal.io/server/common/resolver" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" + rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/testing/testhooks" "google.golang.org/grpc" @@ -61,6 +62,7 @@ type ( persistenceFactoryProvider persistenceClient.FactoryProviderFn searchAttributesMapper searchattribute.Mapper customFrontendInterceptors []grpc.UnaryServerInterceptor + customFrontendNexusInterceptors []rpcinterceptor.NexusInterceptor additionalStreamInterceptors []grpc.StreamServerInterceptor metricHandler metrics.Handler eventLoggerProvider otellog.LoggerProvider From d92d48f097fc2ab5d2e848717bc1bec6a2ae81a3 Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Wed, 12 Aug 2026 16:28:39 -0700 Subject: [PATCH 07/12] add a wrapper for ns rate-limit interceptor --- .../rpc/interceptor/namespace_rate_limit.go | 71 +++++++++++-------- .../interceptor/namespace_rate_limit_test.go | 3 +- service/frontend/fx.go | 1 + .../frontend/nexus_completion_http_handler.go | 4 +- .../frontend/nexus_operation_http_handler.go | 3 +- 5 files changed, 47 insertions(+), 35 deletions(-) diff --git a/common/rpc/interceptor/namespace_rate_limit.go b/common/rpc/interceptor/namespace_rate_limit.go index c72509562fa..7c8db63e8dd 100644 --- a/common/rpc/interceptor/namespace_rate_limit.go +++ b/common/rpc/interceptor/namespace_rate_limit.go @@ -76,8 +76,6 @@ type ( headerGetter headers.HeaderGetter, numToken int, ) error - - InterceptNexus(ctx context.Context, in NexusInterceptorInput, next NexusHandlerFunc) (resp any, err error) } NamespaceRateLimitInterceptorImpl struct { @@ -93,6 +91,46 @@ type ( var _ grpc.UnaryServerInterceptor = (*NamespaceRateLimitInterceptorImpl)(nil).Intercept var _ NamespaceRateLimitInterceptor = (*NamespaceRateLimitInterceptorImpl)(nil) +func NewNexusNamespaceRateLimitInterceptor(ni NamespaceRateLimitInterceptor) *NexusNamespaceRateLimitInterceptor { + return &NexusNamespaceRateLimitInterceptor{ + ni: ni, + } +} + +// NexusNamespaceRateLimitInterceptor is a wrapper on namespace rate limiter +// draft-review: should this interim be removed in favor of a lock step impl w/ deps +type NexusNamespaceRateLimitInterceptor struct { + ni NamespaceRateLimitInterceptor +} + +func (n *NexusNamespaceRateLimitInterceptor) InterceptNexus( + ctx context.Context, + in NexusInterceptorInput, + next NexusHandlerFunc, +) (out any, retErr error) { + apiName, err := NexusAPINameFromContext(ctx) + if err != nil { + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "interceptor_failed", + } + } + header, err := NexusHeaderFromInterceptorInput(in) + if err != nil { + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "interceptor_failed", + } + } + if err := n.ni.Allow(namespace.Name(in.NamespaceName()), apiName, header); err != nil { + return nil, &InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, true), + Outcome: "namespace_rate_limited", + } + } + return next(ctx, in) +} + func NewNamespaceRateLimitInterceptor( namespaceRegistry namespace.Registry, rateLimiter quotas.RequestRateLimiter, @@ -228,35 +266,6 @@ func (ni *NamespaceRateLimitInterceptorImpl) AllowN( return nil } -// InterceptNexus enforces the namespace rate limit for a Nexus request. -func (ni *NamespaceRateLimitInterceptorImpl) InterceptNexus( - ctx context.Context, - in NexusInterceptorInput, - next NexusHandlerFunc, -) (any, error) { - apiName, err := NexusAPINameFromContext(ctx) - if err != nil { - return nil, &InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), - Outcome: "interceptor_failed", - } - } - header, err := NexusHeaderFromInterceptorInput(in) - if err != nil { - return nil, &InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), - Outcome: "interceptor_failed", - } - } - if err := ni.Allow(namespace.Name(in.NamespaceName()), apiName, header); err != nil { - return nil, &InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), - Outcome: "namespace_rate_limited", - } - } - return next(ctx, in) -} - func IsLongPollGetWorkflowExecutionHistoryRequest( req any, ) bool { diff --git a/common/rpc/interceptor/namespace_rate_limit_test.go b/common/rpc/interceptor/namespace_rate_limit_test.go index 11e0911c906..c5e1017118f 100644 --- a/common/rpc/interceptor/namespace_rate_limit_test.go +++ b/common/rpc/interceptor/namespace_rate_limit_test.go @@ -59,7 +59,8 @@ func (s *namespaceRateLimitInterceptorSuite) TestInterceptNexus() { input = NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) } nextCalled := false - _, err := s.newImpl(false).InterceptNexus( + wrapper := NewNexusNamespaceRateLimitInterceptor(s.newImpl(false)) + _, err := wrapper.InterceptNexus( ctx, input, func(context.Context, NexusInterceptorInput) (any, error) { diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 424c17941c9..dfa937990f8 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -131,6 +131,7 @@ var Module = fx.Options( fx.Provide(NewVersionChecker), fx.Provide(ServiceResolverProvider), fx.Provide(newNexusForwardingInterceptor), + fx.Provide(interceptor.NewNexusNamespaceRateLimitInterceptor), fx.Provide(newNexusCompletionHandler), fx.Provide(NewNexusOperationHTTPHandler), fx.Provide(newNexusCompletionHTTPHandler), diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index 235e70482b3..cbc8b89356e 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -76,7 +76,7 @@ func newNexusCompletionHandler( telemetryInterceptor *interceptor.TelemetryInterceptor, requestErrorHandler *interceptor.RequestErrorHandler, namespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor, - namespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor, + nexusNamespaceRateLimitInterceptor *interceptor.NexusNamespaceRateLimitInterceptor, namespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor, rateLimitInterceptor *interceptor.RateLimitInterceptor, authInterceptor *authorization.Interceptor, @@ -94,7 +94,7 @@ func newNexusCompletionHandler( nexusForwarder.InterceptNexus, namespaceValidationInterceptor.InterceptNexus, namespaceConcurrencyLimitInterceptor.InterceptNexus, - namespaceRateLimitInterceptor.InterceptNexus, + nexusNamespaceRateLimitInterceptor.InterceptNexus, rateLimitInterceptor.InterceptNexus, sdkVersionInterceptor.InterceptNexus, callerInfoInterceptor.InterceptNexus, diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index 7e0126ae31b..7dc5a85868e 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -64,6 +64,7 @@ func NewNexusOperationHTTPHandler( redirectionInterceptor *interceptor.Redirection, namespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor, namespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor, + nexusNamespaceRateLimitInterceptor *interceptor.NexusNamespaceRateLimitInterceptor, namespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor, rateLimitInterceptor *interceptor.RateLimitInterceptor, sdkVersionInterceptor *interceptor.SDKVersionInterceptor, @@ -87,7 +88,7 @@ func NewNexusOperationHTTPHandler( nexusForwarder.InterceptNexus, namespaceValidationInterceptor.InterceptNexus, namespaceConcurrencyLimitInterceptor.InterceptNexus, - namespaceRateLimitInterceptor.InterceptNexus, + nexusNamespaceRateLimitInterceptor.InterceptNexus, rateLimitInterceptor.InterceptNexus, sdkVersionInterceptor.InterceptNexus, callerInfoInterceptor.InterceptNexus, From 4259e70fff67e3c50e03f2da0286709ffac2cbe7 Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Thu, 20 Aug 2026 12:07:57 -0700 Subject: [PATCH 08/12] WIP: interceptor refactor- unified interface for ordering --- chasm/interceptors.go | 10 + common/authorization/interceptor.go | 30 +-- common/authorization/interceptor_test.go | 46 +--- common/rpc/interceptor/caller_info.go | 7 +- common/rpc/interceptor/caller_info_test.go | 11 +- .../interceptor/concurrent_request_limit.go | 17 +- .../concurrent_request_limit_test.go | 42 +-- .../context_metadata_interceptor.go | 27 +- .../context_metadata_interceptor_test.go | 2 +- .../rpc/interceptor/frontend_service_error.go | 99 ++++--- .../frontend_service_error_test.go | 2 +- common/rpc/interceptor/health.go | 30 ++- common/rpc/interceptor/mask_internal_error.go | 22 ++ common/rpc/interceptor/namespace.go | 8 + common/rpc/interceptor/namespace_handover.go | 52 ++++ common/rpc/interceptor/namespace_logger.go | 29 +++ .../rpc/interceptor/namespace_rate_limit.go | 39 +-- .../interceptor/namespace_rate_limit_test.go | 21 +- common/rpc/interceptor/namespace_validator.go | 64 +++-- .../interceptor/namespace_validator_test.go | 99 +++---- common/rpc/interceptor/nexus.go | 241 ------------------ common/rpc/interceptor/nexus/nexus.go | 227 +++++++++++++++++ .../rpc/interceptor/{ => nexus}/nexus_test.go | 20 +- common/rpc/interceptor/rate_limit.go | 20 +- common/rpc/interceptor/rate_limit_test.go | 19 +- common/rpc/interceptor/retry.go | 12 + .../interceptor/routing_key_interceptor.go | 18 ++ common/rpc/interceptor/sdk_version.go | 7 +- common/rpc/interceptor/sdk_version_test.go | 7 +- .../interceptor/service_error_interceptor.go | 25 +- common/rpc/interceptor/slow_request_logger.go | 43 +++- common/rpc/interceptor/telemetry.go | 14 +- common/rpc/interceptor/telemetry_test.go | 19 +- service/frontend/frontend_interceptors.go | 207 +++++++++++++++ service/frontend/fx.go | 118 +++++---- service/frontend/fx_test.go | 6 +- .../frontend/nexus_completion_http_handler.go | 102 +++----- service/frontend/nexus_forward_interceptor.go | 59 ++--- .../nexus_forward_interceptor_test.go | 15 +- service/frontend/nexus_handler.go | 80 +++--- service/frontend/nexus_handler_test.go | 31 --- .../frontend/nexus_operation_http_handler.go | 53 ++-- service/fx.go | 2 +- service/history/history_engine_test.go | 6 +- temporal/fx.go | 8 +- temporal/server_option.go | 12 +- temporal/server_options.go | 48 ++-- tools/flakereport/report.go | 6 +- 48 files changed, 1242 insertions(+), 840 deletions(-) delete mode 100644 common/rpc/interceptor/nexus.go create mode 100644 common/rpc/interceptor/nexus/nexus.go rename common/rpc/interceptor/{ => nexus}/nexus_test.go (61%) create mode 100644 service/frontend/frontend_interceptors.go diff --git a/chasm/interceptors.go b/chasm/interceptors.go index 484eb5f0c48..578879fa5f7 100644 --- a/chasm/interceptors.go +++ b/chasm/interceptors.go @@ -5,6 +5,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" + n "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -60,6 +61,15 @@ func (i *ChasmVisibilityInterceptor) Intercept( return handler(ctx, req) } +func (i *ChasmVisibilityInterceptor) InterceptNexus( + ctx context.Context, + in n.InterceptorInput, + next n.HandlerFunc, +) (any, error) { + ctx = NewVisibilityManagerContext(ctx, i.visibilityMgr) + return next(ctx, in) +} + func ChasmVisibilityInterceptorProvider(visibilityMgr VisibilityManager) *ChasmVisibilityInterceptor { return &ChasmVisibilityInterceptor{ visibilityMgr: visibilityMgr, diff --git a/common/authorization/interceptor.go b/common/authorization/interceptor.go index 452713f0feb..9131f8deb48 100644 --- a/common/authorization/interceptor.go +++ b/common/authorization/interceptor.go @@ -19,7 +19,7 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" - "go.temporal.io/server/common/rpc/interceptor" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/rpc/tlsinfo" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -162,34 +162,22 @@ func (a *Interceptor) Intercept( func (a *Interceptor) InterceptNexus( ctx context.Context, - in interceptor.NexusInterceptorInput, - next interceptor.NexusHandlerFunc, + in nexus.InterceptorInput, + next nexus.HandlerFunc, ) (any, error) { a.logger.Debug("authorizing request") if a.authorizer == nil { return next(ctx, in) } namespaceName := in.NamespaceName() - apiName, err := interceptor.NexusAPINameFromContext(ctx) - if err != nil { - return nil, &interceptor.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, false), - Outcome: "interceptor_failed", - } - } - endpointName, err := interceptor.NexusEndpointNameFromContext(ctx) - if err != nil { - return nil, &interceptor.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, false), - Outcome: "interceptor_failed", - } - } + apiName := in.APIName() + endpointName := in.EndpointName() claims, _ := ctx.Value(MappedClaims).(*Claims) //nolint:revive // unchecked-type-assertion: empty claims will 403 // draft-review: check if this might be required to preserve compatibility for custom authorizers // or if its ok since an interface was not already used instead // switch in.(type) { - // case interceptor.StartNexusOpInput, interceptor.CancelNexusOpInput: - // case *interceptor.CancelNexusOpInput: + // case nexus.StartOpInput, nexus.CancelOpInput: + // case *nexus.CancelOpInput: // } ct := &CallTarget{ APIName: apiName, @@ -201,13 +189,13 @@ func (a *Interceptor) InterceptNexus( if err != nil { if permissionDeniedError, ok := errors.AsType[*serviceerror.PermissionDenied](err); ok { a.logger.Debug("Request unauthorized") - return nil, &interceptor.InterceptorError{ + return nil, &nexus.InterceptorError{ Err: commonnexus.AdaptAuthorizeError(permissionDeniedError), Outcome: "unauthorized", } } a.logger.Error("Authorization internal error with processing nexus request", tag.Error(err)) - return nil, &interceptor.InterceptorError{ + return nil, &nexus.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, false), Outcome: "internal_auth_error", } diff --git a/common/authorization/interceptor_test.go b/common/authorization/interceptor_test.go index 99218dd0db9..88c6cfa6743 100644 --- a/common/authorization/interceptor_test.go +++ b/common/authorization/interceptor_test.go @@ -23,7 +23,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" - "go.temporal.io/server/common/rpc/interceptor" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.uber.org/mock/gomock" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -71,14 +71,18 @@ func TestAuthorizerInterceptorSuite(t *testing.T) { } func (s *authorizerInterceptorSuite) TestInterceptNexus() { - input := interceptor.NewStartNexusOpInput( - "service", - "operation", + input := interceptornexus.NewStartOpInput( + "s", + "o", testNamespace, nexus.StartOperationOptions{}, nil, ) apiName, endpoint := "NexusAPI", "endpoint" + input.WithRequestMetadata(interceptornexus.RequestMetadata{ + APIName: apiName, + EndpointName: endpoint, + }) expectedTarget := &CallTarget{ APIName: apiName, NexusEndpointName: endpoint, @@ -93,42 +97,20 @@ func (s *authorizerInterceptorSuite) TestInterceptNexus() { expectedError error }{ { - name: "authorized", - ctx: interceptor.WithNexusEndpointName( - interceptor.WithNexusAPIName(context.Background(), apiName), - endpoint, - ), + name: "authorized", + ctx: context.Background(), authorizationResult: &Result{Decision: DecisionAllow}, nextCalled: true, }, { - name: "unauthorized", - ctx: interceptor.WithNexusEndpointName( - interceptor.WithNexusAPIName(context.Background(), apiName), - endpoint, - ), + name: "unauthorized", + ctx: context.Background(), authorizationResult: &Result{Decision: DecisionDeny}, - expectedError: &interceptor.InterceptorError{ + expectedError: &interceptornexus.InterceptorError{ Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnauthorized, "permission denied"), Outcome: "unauthorized", }, }, - { - name: "missing API name", - ctx: interceptor.WithNexusEndpointName(context.Background(), endpoint), - expectedError: &interceptor.InterceptorError{ - Err: errors.New("nexus API name not found in context"), - Outcome: "interceptor_failed", - }, - }, - { - name: "missing endpoint name", - ctx: interceptor.WithNexusAPIName(context.Background(), apiName), - expectedError: &interceptor.InterceptorError{ - Err: errors.New("nexus endpoint name not found in context"), - Outcome: "interceptor_failed", - }, - }, } { s.Run(tc.name, func() { if tc.authorizationResult != nil { @@ -145,7 +127,7 @@ func (s *authorizerInterceptorSuite) TestInterceptNexus() { _, err := s.interceptor.InterceptNexus( tc.ctx, input, - func(context.Context, interceptor.NexusInterceptorInput) (any, error) { + func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil }) diff --git a/common/rpc/interceptor/caller_info.go b/common/rpc/interceptor/caller_info.go index 30c0e253aed..bcf62d270cf 100644 --- a/common/rpc/interceptor/caller_info.go +++ b/common/rpc/interceptor/caller_info.go @@ -6,6 +6,7 @@ import ( "go.temporal.io/server/common/api" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -43,13 +44,13 @@ func (i *CallerInfoInterceptor) Intercept( // InterceptNexus adds caller information for a Nexus request. func (i *CallerInfoInterceptor) InterceptNexus( ctx context.Context, - in NexusInterceptorInput, - next NexusHandlerFunc, + in nexus.InterceptorInput, + next nexus.HandlerFunc, ) (any, error) { ctx = PopulateCallerInfo( ctx, in.NamespaceName, - func() string { return NexusMethodName(in) }, + func() string { return nexus.MethodName(in) }, ) return next(headers.Propagate(ctx), in) } diff --git a/common/rpc/interceptor/caller_info_test.go b/common/rpc/interceptor/caller_info_test.go index d808ef7bd1c..734214e5071 100644 --- a/common/rpc/interceptor/caller_info_test.go +++ b/common/rpc/interceptor/caller_info_test.go @@ -10,6 +10,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/namespace" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.uber.org/mock/gomock" "google.golang.org/grpc" ) @@ -128,29 +129,29 @@ func (s *callerInfoSuite) TestIntercept_CallerName() { func (s *callerInfoSuite) TestInterceptNexus() { for _, tc := range []struct { name string - input NexusInterceptorInput + input interceptornexus.InterceptorInput callerInfo headers.CallerInfo expectedOrigin string }{ { name: "start", - input: NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + input: interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), expectedOrigin: "StartNexusOperation", }, { name: "cancel - preserves background origin", - input: NewCancelNexusOpInput("s", "o", testNamespace, nexus.CancelOperationOptions{}, "t"), + input: interceptornexus.NewCancelOpInput("s", "o", testNamespace, nexus.CancelOperationOptions{}, "t"), callerInfo: headers.SystemBackgroundHighCallerInfo, }, { name: "complete", - input: NewCompleteNexusOpInput(testNamespace, nil), + input: interceptornexus.NewCompleteOpInput(testNamespace, nil), expectedOrigin: "CompleteNexusOperation", }, } { s.Run(tc.name, func() { ctx := headers.SetCallerInfo(context.Background(), tc.callerInfo) - _, err := s.interceptor.InterceptNexus(ctx, tc.input, func(ctx context.Context, _ NexusInterceptorInput) (any, error) { + _, err := s.interceptor.InterceptNexus(ctx, tc.input, func(ctx context.Context, _ interceptornexus.InterceptorInput) (any, error) { callerInfo := headers.GetCallerInfo(ctx) s.Equal(testNamespace, callerInfo.CallerName) s.Equal(tc.expectedOrigin, callerInfo.CallOrigin) diff --git a/common/rpc/interceptor/concurrent_request_limit.go b/common/rpc/interceptor/concurrent_request_limit.go index 39d5f41a604..f39547894b3 100644 --- a/common/rpc/interceptor/concurrent_request_limit.go +++ b/common/rpc/interceptor/concurrent_request_limit.go @@ -14,6 +14,7 @@ import ( "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/quotas/calculator" + "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -120,22 +121,14 @@ func (ni *ConcurrentRequestLimitInterceptor) Allow( // InterceptNexus enforces the namespace concurrent-request limit for a Nexus request. func (ni *ConcurrentRequestLimitInterceptor) InterceptNexus( ctx context.Context, - in NexusInterceptorInput, - next NexusHandlerFunc, + in nexus.InterceptorInput, + next nexus.HandlerFunc, ) (any, error) { - apiName, err := NexusAPINameFromContext(ctx) - if err != nil { - return nil, &InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, false), - Outcome: "interceptor_failed", - } - } metricsHandler := GetMetricsHandlerFromContext(ctx, ni.logger) - // draft-review: this looks safe to pass "in" as any, but confirm in review - cleanup, err := ni.Allow(namespace.Name(in.NamespaceName()), apiName, metricsHandler, in) + cleanup, err := ni.Allow(namespace.Name(in.NamespaceName()), in.APIName(), metricsHandler, in) defer cleanup() if err != nil { - return nil, &InterceptorError{ + return nil, &nexus.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, false), Outcome: "namespace_concurrency_limited", } diff --git a/common/rpc/interceptor/concurrent_request_limit_test.go b/common/rpc/interceptor/concurrent_request_limit_test.go index cc7199da9e8..f086dd58c0f 100644 --- a/common/rpc/interceptor/concurrent_request_limit_test.go +++ b/common/rpc/interceptor/concurrent_request_limit_test.go @@ -14,6 +14,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/quotas/calculator" "go.temporal.io/server/common/quotas/quotastest" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.uber.org/mock/gomock" "google.golang.org/grpc" ) @@ -151,20 +152,9 @@ func TestConcurrentRequestLimitInterceptor_InterceptNexus(t *testing.T) { dynamicconfig.GetIntPropertyFnFilteredByNamespace(1), map[string]int{"NexusAPI": 1}, ) - input := NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) - t.Run("missing API name", func(t *testing.T) { - nextCalled := false - _, err := interceptor.InterceptNexus(context.Background(), input, func(context.Context, NexusInterceptorInput) (any, error) { - nextCalled = true - return nil, nil - }) - var interceptorErr *InterceptorError - require.ErrorAs(t, err, &interceptorErr) - require.Equal(t, "interceptor_failed", interceptorErr.Outcome) - require.False(t, nextCalled) - }) + input := withAPIName(interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), "NexusAPI") - ctx := WithNexusAPIName(context.Background(), "NexusAPI") + ctx := context.Background() blockUntilFirstReqStarted := make(chan struct{}) unblockFirstRequest := make(chan struct{}) @@ -174,7 +164,7 @@ func TestConcurrentRequestLimitInterceptor_InterceptNexus(t *testing.T) { _, err := interceptor.InterceptNexus( ctx, input, - func(context.Context, NexusInterceptorInput) (any, error) { + func(context.Context, interceptornexus.InterceptorInput) (any, error) { close(blockUntilFirstReqStarted) <-unblockFirstRequest return nil, nil @@ -187,12 +177,12 @@ func TestConcurrentRequestLimitInterceptor_InterceptNexus(t *testing.T) { _, err := interceptor.InterceptNexus( ctx, input, - func(context.Context, NexusInterceptorInput) (any, error) { + func(context.Context, interceptornexus.InterceptorInput) (any, error) { t.Fatal("second request reached handler") return nil, errors.New("throttled request reached") }, ) - var interceptorErr *InterceptorError + var interceptorErr *interceptornexus.InterceptorError require.ErrorAs(t, err, &interceptorErr) require.Equal(t, "namespace_concurrency_limited", interceptorErr.Outcome) @@ -294,3 +284,23 @@ func (h testRequestHandler) Handle(context.Context, any) (any, error) { return nil, nil } + +func withRequestMetadataForTest(in interceptornexus.InterceptorInput, metadata interceptornexus.RequestMetadata) interceptornexus.InterceptorInput { + switch v := in.(type) { + case interceptornexus.StartOpInput: + v.WithRequestMetadata(metadata) + return v + case interceptornexus.CancelOpInput: + v.WithRequestMetadata(metadata) + return v + case interceptornexus.CompleteOpInput: + v.WithRequestMetadata(metadata) + return v + default: + return in + } +} + +func withAPIName(in interceptornexus.InterceptorInput, apiName string) interceptornexus.InterceptorInput { + return withRequestMetadataForTest(in, interceptornexus.RequestMetadata{APIName: apiName}) +} diff --git a/common/rpc/interceptor/context_metadata_interceptor.go b/common/rpc/interceptor/context_metadata_interceptor.go index 1368d417d42..64a685fcc5f 100644 --- a/common/rpc/interceptor/context_metadata_interceptor.go +++ b/common/rpc/interceptor/context_metadata_interceptor.go @@ -8,6 +8,7 @@ import ( "go.temporal.io/server/common/contextutil" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" "google.golang.org/grpc/metadata" "google.golang.org/protobuf/proto" @@ -56,13 +57,29 @@ func (c *ContextMetadataInterceptor) Intercept( resp, err := handler(ctx, req) if c.setTrailer { - c.appendContextMetadataToTrailer(ctx, info) + c.appendContextMetadataToTrailer(ctx, info.FullMethod) } return resp, err } -func (c *ContextMetadataInterceptor) appendContextMetadataToTrailer(ctx context.Context, info *grpc.UnaryServerInfo) { +func (c *ContextMetadataInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + ctx = contextutil.WithMetadataContext(ctx) + + resp, err := next(ctx, in) + + if c.setTrailer { + c.appendContextMetadataToTrailer(ctx, in.APIName()) + } + + return resp, err +} + +func (c *ContextMetadataInterceptor) appendContextMetadataToTrailer(ctx context.Context, method string) { // If the context is done, the gRPC stream may already be in streamDone state, // and SetTrailer would return ErrIllegalHeaderWrite ("SendHeader called multiple times"). select { @@ -74,7 +91,7 @@ func (c *ContextMetadataInterceptor) appendContextMetadataToTrailer(ctx context. allMetadata := contextutil.ContextMetadataGetAll(ctx) if len(allMetadata) == 0 { c.throttledLogger.Info("ContextMetadataInterceptor: No metadata in context, not setting trailer", - tag.NewStringTag("fullMethod", info.FullMethod), + tag.NewStringTag("fullMethod", method), ) return } @@ -84,13 +101,13 @@ func (c *ContextMetadataInterceptor) appendContextMetadataToTrailer(ctx context. trailer := metadata.Pairs(trailerPairs...) c.throttledLogger.Info("ContextMetadataInterceptor: Setting trailer", tag.NewAnyTag("trailer", trailer), - tag.NewStringTag("fullMethod", info.FullMethod), + tag.NewStringTag("fullMethod", method), ) if err := grpc.SetTrailer(ctx, trailer); err != nil { c.logger.Error("ContextMetadataInterceptor: Failed to set trailer", tag.Error(err), - tag.NewStringTag("fullMethod", info.FullMethod)) + tag.NewStringTag("fullMethod", method)) } } diff --git a/common/rpc/interceptor/context_metadata_interceptor_test.go b/common/rpc/interceptor/context_metadata_interceptor_test.go index a359a5b414e..c16cc10a844 100644 --- a/common/rpc/interceptor/context_metadata_interceptor_test.go +++ b/common/rpc/interceptor/context_metadata_interceptor_test.go @@ -197,7 +197,7 @@ func TestContextMetadataInterceptor_appendContextMetadataToTrailer(t *testing.T) info := &grpc.UnaryServerInfo{ FullMethod: "/test.Service/TestMethod", } - interceptor.appendContextMetadataToTrailer(ctx, info) + interceptor.appendContextMetadataToTrailer(ctx, info.FullMethod) }) } } diff --git a/common/rpc/interceptor/frontend_service_error.go b/common/rpc/interceptor/frontend_service_error.go index faf1422b59b..d9dd4229138 100644 --- a/common/rpc/interceptor/frontend_service_error.go +++ b/common/rpc/interceptor/frontend_service_error.go @@ -2,11 +2,13 @@ package interceptor import ( "context" + "errors" "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/api" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/rpc/interceptor/nexus" serviceerrors "go.temporal.io/server/common/serviceerror" "google.golang.org/grpc" "google.golang.org/grpc/metadata" @@ -20,41 +22,74 @@ const ( ResourceExhaustedScopeHeader = "X-Resource-Exhausted-Scope" ) -// NewFrontendServiceErrorInterceptor returns a gRPC interceptor that has two responsibilities: +type FrontendServiceErrorInterceptor struct { + logger log.Logger +} + +// NewFrontendServiceErrorInterceptorWrapper returns interceptors that have two responsibilities: // 1. Mask certain internal service error details. // 2. Propagate resource exhaustion details via gRPC headers. -func NewFrontendServiceErrorInterceptor( - logger log.Logger, -) grpc.UnaryServerInterceptor { - return func( - ctx context.Context, - req any, - info *grpc.UnaryServerInfo, - handler grpc.UnaryHandler, - ) (any, error) { - resp, err := handler(ctx, req) - if err == nil { - return resp, nil - } +func NewFrontendServiceErrorInterceptorWrapper(logger log.Logger) *FrontendServiceErrorInterceptor { + return &FrontendServiceErrorInterceptor{ + logger: logger, + } +} - switch serviceErr := err.(type) { - case *serviceerrors.ShardOwnershipLost: - err = serviceerror.NewUnavailable("shard unavailable, please backoff and retry") - case *serviceerror.DataLoss: - err = serviceerror.NewUnavailable("internal history service error") - case *serviceerror.ResourceExhausted: - if headerErr := grpc.SetHeader(ctx, metadata.Pairs( - ResourceExhaustedCauseHeader, serviceErr.Cause.String(), - ResourceExhaustedScopeHeader, serviceErr.Scope.String(), - )); headerErr != nil { - // So while this is *not* a user-facing error or problem in itself, - // it indicates that there might be larger connection issues at play. - logger.Error("Failed to add Resource-Exhausted headers to response", - tag.Operation(api.MethodName(info.FullMethod)), - tag.Error(headerErr)) - } - } +func NewFrontendServiceErrorInterceptor(logger log.Logger) grpc.UnaryServerInterceptor { + t := NewFrontendServiceErrorInterceptorWrapper(logger) + return t.Intercept +} - return resp, err +func (f *FrontendServiceErrorInterceptor) Intercept( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, +) (any, error) { + resp, err := handler(ctx, req) + + return resp, f.transformError(ctx, info.FullMethod, err, true) +} + +func (f *FrontendServiceErrorInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + resp, err := next(ctx, in) + if ie, ok := errors.AsType[*nexus.InterceptorError](err); ok { + ie.Err = f.transformError(ctx, in.APIName(), ie.Err, false) + return resp, ie + } + return resp, f.transformError(ctx, in.APIName(), err, false) +} + +func (f *FrontendServiceErrorInterceptor) transformError(ctx context.Context, method string, err error, isGRPC bool) error { + if err == nil { + return nil + } + method = api.MethodName(method) + + switch serviceErr := err.(type) { + case *serviceerrors.ShardOwnershipLost: + err = serviceerror.NewUnavailable("shard unavailable, please backoff and retry") + case *serviceerror.DataLoss: + err = serviceerror.NewUnavailable("internal history service error") + case *serviceerror.ResourceExhausted: + if !isGRPC { + break + } + if headerErr := grpc.SetHeader(ctx, metadata.Pairs( + ResourceExhaustedCauseHeader, serviceErr.Cause.String(), + ResourceExhaustedScopeHeader, serviceErr.Scope.String(), + )); headerErr != nil { + // So while this is *not* a user-facing error or problem in itself, + // it indicates that there might be larger connection issues at play. + f.logger.Error("Failed to add Resource-Exhausted headers to response", + tag.Operation(method), + tag.Error(headerErr)) + } + default: } + return err } diff --git a/common/rpc/interceptor/frontend_service_error_test.go b/common/rpc/interceptor/frontend_service_error_test.go index f50e0e64873..b55ff690491 100644 --- a/common/rpc/interceptor/frontend_service_error_test.go +++ b/common/rpc/interceptor/frontend_service_error_test.go @@ -105,7 +105,7 @@ func TestFrontendServiceErrorInterceptor(t *testing.T) { } ctx := grpc.NewContextWithServerTransportStream(context.Background(), stream) - var interceptorFn = NewFrontendServiceErrorInterceptor(tl) + var interceptorFn = NewFrontendServiceErrorInterceptorWrapper(tl).Intercept info := &grpc.UnaryServerInfo{FullMethod: method} _, err := interceptorFn(ctx, nil, info, func(_ context.Context, _ any) (any, error) { diff --git a/common/rpc/interceptor/health.go b/common/rpc/interceptor/health.go index 3023dd6ff5a..d7f5ae7a0a3 100644 --- a/common/rpc/interceptor/health.go +++ b/common/rpc/interceptor/health.go @@ -7,6 +7,7 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/api" + "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -32,16 +33,33 @@ func (i *HealthInterceptor) Intercept( info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ) (any, error) { - // only enforce health check on WorkflowService and OperatorService - if strings.HasPrefix(info.FullMethod, api.WorkflowServicePrefix) || - strings.HasPrefix(info.FullMethod, api.OperatorServicePrefix) { - if !i.healthy.Load() { - return nil, notHealthyErr - } + if i.isNotHealthy(info.FullMethod) { + return nil, notHealthyErr } return handler(ctx, req) } +func (i *HealthInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + if i.isNotHealthy(in.OperationName()) { + return nil, notHealthyErr + } + return next(ctx, in) +} + +func (i *HealthInterceptor) isNotHealthy(methodName string) bool { + if i.healthy.Load() { + return false + } + + // only enforce health check on WorkflowService and OperatorService + return strings.HasPrefix(methodName, api.WorkflowServicePrefix) || + strings.HasPrefix(methodName, api.OperatorServicePrefix) +} + func (i *HealthInterceptor) SetHealthy(healthy bool) { i.healthy.Store(healthy) } diff --git a/common/rpc/interceptor/mask_internal_error.go b/common/rpc/interceptor/mask_internal_error.go index 0676dcdbb96..e8d99402afd 100644 --- a/common/rpc/interceptor/mask_internal_error.go +++ b/common/rpc/interceptor/mask_internal_error.go @@ -2,6 +2,7 @@ package interceptor import ( "context" + "errors" "fmt" "go.temporal.io/api/serviceerror" @@ -12,6 +13,7 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/rpc/interceptor/logtags" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/tasktoken" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -56,6 +58,26 @@ func (mi *MaskInternalErrorDetailsInterceptor) Intercept( return resp, err } +func (mi *MaskInternalErrorDetailsInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + + resp, err := next(ctx, in) + + if err == nil || !mi.shouldMaskErrors(in) { + return resp, err + } + if ie, ok := errors.AsType[*nexus.InterceptorError](err); ok { + ie.Err = mi.maskUnknownOrInternalErrors(in, in.APIName(), ie.Err) + err = ie + } else { + err = mi.maskUnknownOrInternalErrors(in, in.APIName(), err) + } + return resp, err +} + func (mi *MaskInternalErrorDetailsInterceptor) shouldMaskErrors(req any) bool { ns := MustGetNamespaceName(mi.namespaceRegistry, req) if ns.IsEmpty() { diff --git a/common/rpc/interceptor/namespace.go b/common/rpc/interceptor/namespace.go index fdeded48add..11ba12a25b0 100644 --- a/common/rpc/interceptor/namespace.go +++ b/common/rpc/interceptor/namespace.go @@ -4,6 +4,7 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/interceptor/nexus" ) // gRPC method request must implement either NamespaceNameGetter or NamespaceIDGetter @@ -57,6 +58,13 @@ func GetNamespaceName( } return namespaceName, nil + case nexus.InterceptorInput: + ns, err := request.NamespaceEntry() + if err != nil { + return namespace.EmptyName, err + } + return ns.Name(), nil + default: return namespace.EmptyName, serviceerror.NewInternalf("unable to extract namespace info from request of type %T", req) } diff --git a/common/rpc/interceptor/namespace_handover.go b/common/rpc/interceptor/namespace_handover.go index 4d1962d683a..f3649648720 100644 --- a/common/rpc/interceptor/namespace_handover.go +++ b/common/rpc/interceptor/namespace_handover.go @@ -14,6 +14,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -91,6 +92,57 @@ func (i *NamespaceHandoverInterceptor) handlesMethod(fullMethod string) bool { return false } +// draft-review: this looks correct, but check in review +// If this is the right way, extract the common logic into a util +// +//nolint:staticcheck +func (i *NamespaceHandoverInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (_ any, retError error) { + defer log.CapturePanic(i.logger, &retError) + + apiName := in.APIName() + if !i.handlesMethod(apiName) { + return next(ctx, in) + } + methodName := api.MethodName(apiName) + namespaceName := MustGetNamespaceName(i.namespaceRegistry, in) + + if namespaceName != namespace.EmptyName { + var waitTime *time.Duration + defer func() { + if waitTime != nil { + metrics.HandoverWaitLatency.With(i.metricsHandler).Record(*waitTime) + } + }() + waitTime, err := i.waitNamespaceHandoverUpdate(ctx, namespaceName, methodName) + if err != nil { + metricsHandler, logTags := CreateUnaryMetricsHandlerLogTags( + i.metricsHandler, + in, + apiName, + methodName, + namespaceName, + ) + // count the request as this will not be counted + metrics.ServiceRequests.With(metricsHandler).Record(1) + + i.requestErrorHandler.HandleError( + in, + apiName, + metricsHandler, + logTags, + err, + namespaceName, + ) + return nil, err + } + } + return next(ctx, in) +} + func (i *NamespaceHandoverInterceptor) Intercept( ctx context.Context, req any, diff --git a/common/rpc/interceptor/namespace_logger.go b/common/rpc/interceptor/namespace_logger.go index 65b4698839f..9b2365fd006 100644 --- a/common/rpc/interceptor/namespace_logger.go +++ b/common/rpc/interceptor/namespace_logger.go @@ -9,6 +9,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/rpc/tlsinfo" "google.golang.org/grpc" ) @@ -59,3 +60,31 @@ func (nli *NamespaceLogInterceptor) Intercept( } return handler(ctx, req) } + +func (nli *NamespaceLogInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + if nli.logger != nil { + methodName := api.MethodName(in.APIName()) + namespaceName := MustGetNamespaceName(nli.namespaceRegistry, in) + tlsInfo := tlsinfo.FromContext(ctx) + var serverName string + var certThumbprint string + if tlsInfo != nil { + serverName = tlsInfo.State.ServerName + cert := tlsinfo.PeerCert(tlsInfo) + if cert != nil { + certThumbprint = fmt.Sprintf("%x", md5.Sum(cert.Raw)) + } + } + nli.logger.Debug( + "Frontend method invoked.", + tag.WorkflowNamespace(namespaceName.String()), + tag.Operation(methodName), + tag.ServerName(serverName), + tag.CertThumbprint(certThumbprint)) + } + return next(ctx, in) +} diff --git a/common/rpc/interceptor/namespace_rate_limit.go b/common/rpc/interceptor/namespace_rate_limit.go index 7c8db63e8dd..f04e4e68d33 100644 --- a/common/rpc/interceptor/namespace_rate_limit.go +++ b/common/rpc/interceptor/namespace_rate_limit.go @@ -14,6 +14,7 @@ import ( "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/quotas" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/service/frontend/configs" "google.golang.org/grpc" ) @@ -91,39 +92,41 @@ type ( var _ grpc.UnaryServerInterceptor = (*NamespaceRateLimitInterceptorImpl)(nil).Intercept var _ NamespaceRateLimitInterceptor = (*NamespaceRateLimitInterceptorImpl)(nil) -func NewNexusNamespaceRateLimitInterceptor(ni NamespaceRateLimitInterceptor) *NexusNamespaceRateLimitInterceptor { - return &NexusNamespaceRateLimitInterceptor{ +func NewNamespaceRateLimitInterceptorWrapper(ni NamespaceRateLimitInterceptor) *NamespaceRateLimitInterceptorWrapper { + return &NamespaceRateLimitInterceptorWrapper{ ni: ni, } } -// NexusNamespaceRateLimitInterceptor is a wrapper on namespace rate limiter +// NamespaceRateLimitInterceptorWrapper is a wrapper on namespace rate limiter // draft-review: should this interim be removed in favor of a lock step impl w/ deps -type NexusNamespaceRateLimitInterceptor struct { +type NamespaceRateLimitInterceptorWrapper struct { ni NamespaceRateLimitInterceptor } -func (n *NexusNamespaceRateLimitInterceptor) InterceptNexus( +func (n *NamespaceRateLimitInterceptorWrapper) Intercept( ctx context.Context, - in NexusInterceptorInput, - next NexusHandlerFunc, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, +) (resp any, err error) { + return n.ni.Intercept(ctx, req, info, handler) +} + +func (n *NamespaceRateLimitInterceptorWrapper) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, ) (out any, retErr error) { - apiName, err := NexusAPINameFromContext(ctx) - if err != nil { - return nil, &InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), - Outcome: "interceptor_failed", - } - } - header, err := NexusHeaderFromInterceptorInput(in) + header, err := nexus.HeaderFromInterceptorInput(in) if err != nil { - return nil, &InterceptorError{ + return nil, &nexus.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, true), Outcome: "interceptor_failed", } } - if err := n.ni.Allow(namespace.Name(in.NamespaceName()), apiName, header); err != nil { - return nil, &InterceptorError{ + if err := n.ni.Allow(namespace.Name(in.NamespaceName()), in.APIName(), header); err != nil { + return nil, &nexus.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, true), Outcome: "namespace_rate_limited", } diff --git a/common/rpc/interceptor/namespace_rate_limit_test.go b/common/rpc/interceptor/namespace_rate_limit_test.go index c5e1017118f..edbb0604d3f 100644 --- a/common/rpc/interceptor/namespace_rate_limit_test.go +++ b/common/rpc/interceptor/namespace_rate_limit_test.go @@ -13,6 +13,7 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/quotas" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.uber.org/mock/gomock" "google.golang.org/grpc" ) @@ -36,40 +37,36 @@ func (s *namespaceRateLimitInterceptorSuite) TestInterceptNexus() { for _, tc := range []struct { name string apiName string - input NexusInterceptorInput + input interceptornexus.InterceptorInput allow *bool nextCalled bool expectedOutcome string }{ - {name: "allowed", apiName: "NexusOperation", input: NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), allow: new(true), nextCalled: true}, - {name: "rate limited", apiName: "NexusOperation", input: NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), allow: new(false), expectedOutcome: "namespace_rate_limited"}, - {name: "missing API name", expectedOutcome: "interceptor_failed"}, - {name: "missing request header", apiName: "NexusOperation", input: NewCompleteNexusOpInput(testNamespace, nil), expectedOutcome: "interceptor_failed"}, + {name: "allowed", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), "NexusOperation"), allow: new(true), nextCalled: true}, + {name: "rate limited", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), "NexusOperation"), allow: new(false), expectedOutcome: "namespace_rate_limited"}, + {name: "missing request header", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewCompleteOpInput(testNamespace, nil), "NexusOperation"), expectedOutcome: "interceptor_failed"}, } { s.Run(tc.name, func() { ctx := context.Background() - if tc.apiName != "" { - ctx = WithNexusAPIName(ctx, tc.apiName) - } if tc.allow != nil { s.mockRateLimiter.EXPECT().Allow(gomock.Any(), gomock.Any()).Return(*tc.allow) } input := tc.input if input == nil { - input = NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) + input = interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) } nextCalled := false - wrapper := NewNexusNamespaceRateLimitInterceptor(s.newImpl(false)) + wrapper := NewNamespaceRateLimitInterceptorWrapper(s.newImpl(false)) _, err := wrapper.InterceptNexus( ctx, input, - func(context.Context, NexusInterceptorInput) (any, error) { + func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil }, ) if tc.expectedOutcome != "" { - var interceptorErr *InterceptorError + var interceptorErr *interceptornexus.InterceptorError s.ErrorAs(err, &interceptorErr) s.Equal(tc.expectedOutcome, interceptorErr.Outcome) } else { diff --git a/common/rpc/interceptor/namespace_validator.go b/common/rpc/interceptor/namespace_validator.go index 54771e6b680..2c5c83fe47c 100644 --- a/common/rpc/interceptor/namespace_validator.go +++ b/common/rpc/interceptor/namespace_validator.go @@ -13,6 +13,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/tasktoken" "google.golang.org/grpc" ) @@ -30,6 +31,13 @@ type ( maxNamespaceLength dynamicconfig.IntPropertyFn additionalAllowedMethodsDuringHandover map[string]struct{} } + + // NamespaceStateValidatorInterceptor contains NamespaceValidatorInterceptor to validate state. + // It is separate from NamespaceValidatorInterceptor to allow both to expose cleaner + // Intercept/InterceptNexus methods that are used as gRPC and Nexus interceptors + NamespaceStateValidatorInterceptor struct { + nvi *NamespaceValidatorInterceptor + } ) var ( @@ -87,8 +95,8 @@ var ( } ) -var _ grpc.UnaryServerInterceptor = (*NamespaceValidatorInterceptor)(nil).StateValidationIntercept -var _ grpc.UnaryServerInterceptor = (*NamespaceValidatorInterceptor)(nil).NamespaceValidateIntercept +var _ grpc.UnaryServerInterceptor = (*NamespaceValidatorInterceptor)(nil).Intercept +var _ grpc.UnaryServerInterceptor = (*NamespaceStateValidatorInterceptor)(nil).Intercept func NewNamespaceValidatorInterceptor( namespaceRegistry namespace.Registry, @@ -109,12 +117,19 @@ func NewNamespaceValidatorInterceptor( } } -func (ni *NamespaceValidatorInterceptor) NamespaceValidateIntercept( +func NewNamespaceStateValidatorInterceptor(nvi *NamespaceValidatorInterceptor) *NamespaceStateValidatorInterceptor { + return &NamespaceStateValidatorInterceptor{ + nvi: nvi, + } +} + +func (nsvi *NamespaceStateValidatorInterceptor) Intercept( ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ) (any, error) { + ni := nsvi.nvi err := ni.setNamespaceIfNotPresent(req) if err != nil { return nil, err @@ -129,6 +144,26 @@ func (ni *NamespaceValidatorInterceptor) NamespaceValidateIntercept( return handler(ctx, req) } +func (nsvi *NamespaceStateValidatorInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + ni := nsvi.nvi + ns, err := in.NamespaceEntry() + if err != nil { + return nil, &nexus.InterceptorError{ + Err: commonnexus.ConvertGRPCError(err, false), + Outcome: "interceptor_failed", + } + } + if err := ni.ValidateName(ns.Info().GetName()); err != nil { + return nil, err + } + + return next(ctx, in) +} + // ValidateName validates a namespace name (currently only a max length check). func (ni *NamespaceValidatorInterceptor) ValidateName(ns string) error { if len(ns) > ni.maxNamespaceLength() { @@ -199,8 +234,8 @@ func (ni *NamespaceValidatorInterceptor) setNamespace( } } -// StateValidationIntercept runs ValidateState - see docstring for that method. -func (ni *NamespaceValidatorInterceptor) StateValidationIntercept( +// Intercept runs ValidateState - see docstring for that method. +func (ni *NamespaceValidatorInterceptor) Intercept( ctx context.Context, req any, info *grpc.UnaryServerInfo, @@ -234,25 +269,18 @@ func (ni *NamespaceValidatorInterceptor) ValidateState(namespaceEntry *namespace // InterceptNexus validates the namespace state for a Nexus request. func (ni *NamespaceValidatorInterceptor) InterceptNexus( ctx context.Context, - in NexusInterceptorInput, - next NexusHandlerFunc, + in nexus.InterceptorInput, + next nexus.HandlerFunc, ) (any, error) { - namespaceEntry, err := NexusNamespaceFromContext(ctx) - if err != nil { - return nil, &InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, false), - Outcome: "interceptor_failed", - } - } - apiName, err := NexusAPINameFromContext(ctx) + namespaceEntry, err := in.NamespaceEntry() if err != nil { - return nil, &InterceptorError{ + return nil, &nexus.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, false), Outcome: "interceptor_failed", } } - if err := ni.ValidateState(namespaceEntry, apiName, in.ForwardingInfo().BusinessID); err != nil { - return nil, &InterceptorError{ + if err := ni.ValidateState(namespaceEntry, in.APIName(), in.ForwardingInfo().BusinessID); err != nil { + return nil, &nexus.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, false), Outcome: "invalid_namespace_state", } diff --git a/common/rpc/interceptor/namespace_validator_test.go b/common/rpc/interceptor/namespace_validator_test.go index 9017821af90..d0f1776478b 100644 --- a/common/rpc/interceptor/namespace_validator_test.go +++ b/common/rpc/interceptor/namespace_validator_test.go @@ -20,6 +20,7 @@ import ( "go.temporal.io/server/common/api" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/namespace" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/tasktoken" "go.uber.org/mock/gomock" "google.golang.org/grpc" @@ -97,7 +98,7 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_NamespaceNotSet( for _, testCase := range testCases { handlerCalled := false - _, err := nvi.StateValidationIntercept(context.Background(), testCase.req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err := nvi.Intercept(context.Background(), testCase.req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.StartWorkflowExecutionResponse{}, nil }) @@ -119,60 +120,67 @@ func (s *namespaceValidatorSuite) TestInterceptNexus() { dynamicconfig.GetIntPropertyFn(100), nil, ) - input := NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) for _, tc := range []struct { name string - ctx context.Context + input interceptornexus.InterceptorInput nextCalled bool expectedOutcome string }{ { name: "resolved namespace", - ctx: WithNexusAPIName(WithNexusNamespace(context.Background(), namespace.NewNamespaceForTest( - &persistencespb.NamespaceInfo{Name: testNamespace, State: enumspb.NAMESPACE_STATE_REGISTERED}, - nil, - false, - nil, - 0, - )), api.NexusServicePrefix+"DispatchNexusTask"), + input: withRequestMetadataForTest( + interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + interceptornexus.RequestMetadata{ + APIName: api.NexusServicePrefix + "DispatchNexusTask", + NamespaceEntry: namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace, State: enumspb.NAMESPACE_STATE_REGISTERED}, + nil, + false, + nil, + 0, + ), + }, + ), nextCalled: true, }, { name: "invalid namespace state", - ctx: WithNexusAPIName(WithNexusNamespace(context.Background(), namespace.NewNamespaceForTest( - &persistencespb.NamespaceInfo{Name: testNamespace, State: enumspb.NAMESPACE_STATE_DEPRECATED}, - nil, - false, - nil, - 0, - )), api.NexusServicePrefix+"DispatchNexusTask"), + input: withRequestMetadataForTest( + interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + interceptornexus.RequestMetadata{ + APIName: api.NexusServicePrefix + "DispatchNexusTask", + NamespaceEntry: namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace, State: enumspb.NAMESPACE_STATE_DEPRECATED}, + nil, + false, + nil, + 0, + ), + }, + ), expectedOutcome: "invalid_namespace_state", }, - {name: "missing namespace", ctx: WithNexusAPIName(context.Background(), "NexusAPI"), expectedOutcome: "interceptor_failed"}, { - name: "missing API name", - ctx: WithNexusNamespace(context.Background(), namespace.NewNamespaceForTest( - &persistencespb.NamespaceInfo{Name: testNamespace, State: enumspb.NAMESPACE_STATE_REGISTERED}, - nil, - false, - nil, - 0, - )), + name: "missing namespace", + input: withAPIName( + interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + "NexusAPI", + ), expectedOutcome: "interceptor_failed", }, } { s.Run(tc.name, func() { nextCalled := false _, err := validator.InterceptNexus( - tc.ctx, - input, - func(context.Context, NexusInterceptorInput) (any, error) { + context.Background(), + tc.input, + func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil }, ) if tc.expectedOutcome != "" { - var interceptorErr *InterceptorError + var interceptorErr *interceptornexus.InterceptorError s.ErrorAs(err, &interceptorErr) s.Equal(tc.expectedOutcome, interceptorErr.Outcome) } else { @@ -197,7 +205,7 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_NamespaceNotFoun s.mockRegistry.EXPECT().GetNamespace(namespace.Name("not-found-namespace")).Return(nil, serviceerror.NewNamespaceNotFound("missing-namespace")) req := &workflowservice.StartWorkflowExecutionRequest{Namespace: "not-found-namespace"} handlerCalled := false - _, err := nvi.StateValidationIntercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err := nvi.Intercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.StartWorkflowExecutionResponse{}, nil }) @@ -214,7 +222,7 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_NamespaceNotFoun TaskToken: taskToken, } handlerCalled = false - _, err = nvi.StateValidationIntercept(context.Background(), tokenReq, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err = nvi.Intercept(context.Background(), tokenReq, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.RespondWorkflowTaskCompletedResponse{}, nil }) @@ -471,7 +479,7 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_StatusFromNamesp } handlerCalled := false - _, err := nvi.StateValidationIntercept(context.Background(), testCase.req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err := nvi.Intercept(context.Background(), testCase.req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.StartWorkflowExecutionResponse{}, nil }) @@ -548,7 +556,7 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_StatusFromToken( } handlerCalled := false - _, err := nvi.StateValidationIntercept(context.Background(), testCase.req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err := nvi.Intercept(context.Background(), testCase.req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.RespondWorkflowTaskCompletedResponse{}, nil }) @@ -575,7 +583,7 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_DescribeNamespac req := &workflowservice.DescribeNamespaceRequest{Id: "test-namespace-id"} handlerCalled := false - _, err := nvi.StateValidationIntercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err := nvi.Intercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.DescribeNamespaceResponse{}, nil }) @@ -585,7 +593,7 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_DescribeNamespac req = &workflowservice.DescribeNamespaceRequest{} handlerCalled = false - _, err = nvi.StateValidationIntercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err = nvi.Intercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.DescribeNamespaceResponse{}, nil }) @@ -607,7 +615,7 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_GetClusterInfo() // Example of API which doesn't have namespace field. req := &workflowservice.GetClusterInfoRequest{} handlerCalled := false - _, err := nvi.StateValidationIntercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err := nvi.Intercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.GetClusterInfoResponse{}, nil }) @@ -628,7 +636,7 @@ func (s *namespaceValidatorSuite) Test_Intercept_RegisterNamespace() { req := &workflowservice.RegisterNamespaceRequest{Namespace: "new-namespace"} handlerCalled := false - _, err := nvi.StateValidationIntercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err := nvi.Intercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.RegisterNamespaceResponse{}, nil }) @@ -638,7 +646,7 @@ func (s *namespaceValidatorSuite) Test_Intercept_RegisterNamespace() { req = &workflowservice.RegisterNamespaceRequest{} handlerCalled = false - _, err = nvi.StateValidationIntercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err = nvi.Intercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.RegisterNamespaceResponse{}, nil }) @@ -741,11 +749,11 @@ func (s *namespaceValidatorSuite) Test_StateValidationIntercept_TokenNamespaceEn } handlerCalled := false - _, err = nvi.StateValidationIntercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err = nvi.Intercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.RespondWorkflowTaskCompletedResponse{}, nil }) - _, queryErr := nvi.StateValidationIntercept(context.Background(), queryReq, serverInfo, func(ctx context.Context, req any) (any, error) { + _, queryErr := nvi.Intercept(context.Background(), queryReq, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.RespondQueryTaskCompletedResponse{}, nil }) @@ -787,7 +795,7 @@ func (s *namespaceValidatorSuite) Test_Intercept_DescribeHistoryHostRequests() { } handlerCalled := false - _, err := nvi.StateValidationIntercept( + _, err := nvi.Intercept( context.Background(), testCase.req, serverInfo, @@ -873,7 +881,7 @@ func (s *namespaceValidatorSuite) Test_Intercept_SearchAttributeRequests() { } handlerCalled := false - _, err := nvi.StateValidationIntercept( + _, err := nvi.Intercept( context.Background(), testCase.req, serverInfo, @@ -893,6 +901,7 @@ func (s *namespaceValidatorSuite) Test_NamespaceValidateIntercept() { dynamicconfig.GetBoolPropertyFn(false), dynamicconfig.GetIntPropertyFn(10), nil) + nnvi := NewNamespaceStateValidatorInterceptor(nvi) serverInfo := &grpc.UnaryServerInfo{ FullMethod: api.WorkflowServicePrefix + "random", } @@ -924,7 +933,7 @@ func (s *namespaceValidatorSuite) Test_NamespaceValidateIntercept() { req := &workflowservice.StartWorkflowExecutionRequest{Namespace: "namespace"} handlerCalled := false - _, err = nvi.NamespaceValidateIntercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err = nnvi.Intercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.StartWorkflowExecutionResponse{}, nil }) @@ -933,7 +942,7 @@ func (s *namespaceValidatorSuite) Test_NamespaceValidateIntercept() { req = &workflowservice.StartWorkflowExecutionRequest{Namespace: "namespaceTooLong"} handlerCalled = false - _, err = nvi.NamespaceValidateIntercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { + _, err = nnvi.Intercept(context.Background(), req, serverInfo, func(ctx context.Context, req any) (any, error) { handlerCalled = true return &workflowservice.StartWorkflowExecutionResponse{}, nil }) diff --git a/common/rpc/interceptor/nexus.go b/common/rpc/interceptor/nexus.go deleted file mode 100644 index 4570485c3dc..00000000000 --- a/common/rpc/interceptor/nexus.go +++ /dev/null @@ -1,241 +0,0 @@ -package interceptor - -import ( - "context" - "errors" - "net/http" - "slices" - - "github.com/nexus-rpc/sdk-go/nexus" - "go.temporal.io/server/common/headers" - "go.temporal.io/server/common/namespace" - "go.temporal.io/server/common/nexus/nexusrpc" -) - -type NexusHandlerFunc func(ctx context.Context, in NexusInterceptorInput) (any, error) - -type NexusInterceptor func(ctx context.Context, in NexusInterceptorInput, next NexusHandlerFunc) (any, error) - -type NexusInterceptorInput interface { - ServiceName() string - OperationName() string - NamespaceName() string - ForwardingInfo() NexusForwardingInfo - sealNexusOp() -} - -var ( - _ NexusInterceptorInput = StartNexusOpInput{} - _ NexusInterceptorInput = CancelNexusOpInput{} - _ NexusInterceptorInput = CompleteNexusOpInput{} -) - -// NexusForwardingInfo contains the request data needed to forward a Nexus operation. -type NexusForwardingInfo struct { - OriginalRequestHeaders http.Header - TaskQueue string - EndpointID string - EndpointName string - BusinessID string -} - -type InterceptorError struct { - // wrapped error - Err error - // Outcome tag for metrics reporting, (draft-review: should Outcomes be enum or at least constants instead) - Outcome string -} - -func (t *InterceptorError) Error() string { - return t.Err.Error() -} - -func (t *InterceptorError) Unwrap() error { - return t.Err -} - -// container for ServiceName(), OperationName(), NamespaceName(), ForwardingInfo() -type nexusOpBase struct { - serviceName, operation, namespaceName string - forwardingInfo NexusForwardingInfo -} - -func (b *nexusOpBase) WithForwardingInfo(info NexusForwardingInfo) { - b.forwardingInfo = info -} - -func (b nexusOpBase) ServiceName() string { - return b.serviceName -} - -func (b nexusOpBase) OperationName() string { - return b.operation -} - -func (b nexusOpBase) NamespaceName() string { - return b.namespaceName -} - -func (b nexusOpBase) ForwardingInfo() NexusForwardingInfo { - return b.forwardingInfo -} - -func (nexusOpBase) sealNexusOp() {} - -func NexusHeaderFromInterceptorInput(in NexusInterceptorInput) (headers.HeaderGetter, error) { - switch opts := in.(type) { - case StartNexusOpInput: - return opts.StartOperationOptions.Header, nil - case CancelNexusOpInput: - return opts.CancelOperationOptions.Header, nil - case CompleteNexusOpInput: - if opts.CompletionRequest == nil || opts.CompletionRequest.HTTPRequest == nil { - return nil, errors.New("nexus completion request not found") - } - return opts.CompletionRequest.HTTPRequest.Header, nil - default: - return nil, errors.New("unknown Nexus interceptor input") - } -} - -// draft-review: verify that these are the "right" methods/names -// -//nolint:staticcheck -func NexusMethodName(in NexusInterceptorInput) string { - switch in.(type) { - case StartNexusOpInput: - return "StartNexusOperation" - case CancelNexusOpInput: - return "CancelNexusOperation" - case CompleteNexusOpInput: - return "CompleteNexusOperation" - default: - return "" - } -} - -type StartNexusOpInput struct { - nexusOpBase - StartOperationOptions nexus.StartOperationOptions - StartOperationInput *nexus.LazyValue -} - -func NewStartNexusOpInput( - serviceName string, - operation string, - namespaceName string, - options nexus.StartOperationOptions, - input *nexus.LazyValue, -) StartNexusOpInput { - return StartNexusOpInput{ - nexusOpBase: nexusOpBase{ - serviceName: serviceName, - operation: operation, - namespaceName: namespaceName, - }, - StartOperationOptions: options, - StartOperationInput: input, - } -} - -type CancelNexusOpInput struct { - nexusOpBase - CancelOperationOptions nexus.CancelOperationOptions - CancellationToken string -} - -func NewCancelNexusOpInput( - serviceName string, - operation string, - namespaceName string, - options nexus.CancelOperationOptions, - cancellationToken string, -) CancelNexusOpInput { - return CancelNexusOpInput{ - nexusOpBase: nexusOpBase{ - serviceName: serviceName, - operation: operation, - namespaceName: namespaceName, - }, - CancelOperationOptions: options, - CancellationToken: cancellationToken, - } -} - -type CompleteNexusOpInput struct { - nexusOpBase - CompletionRequest *nexusrpc.CompletionRequest -} - -// draft-review: Complete doesnt need servicename/op - verify -// -//nolint:staticcheck -func NewCompleteNexusOpInput( - namespaceName string, - request *nexusrpc.CompletionRequest, -) CompleteNexusOpInput { - return CompleteNexusOpInput{ - nexusOpBase: nexusOpBase{ - namespaceName: namespaceName, - }, - CompletionRequest: request, - } -} - -func ChainNexusInterceptors(final NexusHandlerFunc, chain []NexusInterceptor) NexusHandlerFunc { - for _, curr := range slices.Backward(chain) { - next := final - final = func(ctx context.Context, opts NexusInterceptorInput) (any, error) { - return curr(ctx, opts, next) - } - } - return final -} - -type nexusAPINameContextKey struct{} -type nexusEndpointNameContextKey struct{} - -// draft-review: only endpoint and apiName are unknowable - the namespace we should be able to get via lookup -type nexusNamespaceContextKey struct{} - -// WithNexusAPIName adds the internal Nexus API name to a request context. -func WithNexusAPIName(ctx context.Context, apiName string) context.Context { - return context.WithValue(ctx, nexusAPINameContextKey{}, apiName) -} - -func NexusAPINameFromContext(ctx context.Context) (string, error) { - apiName, ok := ctx.Value(nexusAPINameContextKey{}).(string) - if !ok { - return "", errors.New("nexus API name not found in context") - } - return apiName, nil -} - -// WithNexusEndpointName adds the resolved Nexus endpoint name to a request context. -func WithNexusEndpointName(ctx context.Context, endpointName string) context.Context { - return context.WithValue(ctx, nexusEndpointNameContextKey{}, endpointName) -} - -func NexusEndpointNameFromContext(ctx context.Context) (string, error) { - endpointName, ok := ctx.Value(nexusEndpointNameContextKey{}).(string) - if !ok { - return "", errors.New("nexus endpoint name not found in context") - } - return endpointName, nil -} - -// WithNexusNamespace adds the resolved namespace to a request context. -func WithNexusNamespace(ctx context.Context, namespaceEntry *namespace.Namespace) context.Context { - return context.WithValue(ctx, nexusNamespaceContextKey{}, namespaceEntry) -} - -// draft-review: ideally, there is some utility to lookup by name -> Namespace -// -//nolint:staticcheck -func NexusNamespaceFromContext(ctx context.Context) (*namespace.Namespace, error) { - namespaceEntry, ok := ctx.Value(nexusNamespaceContextKey{}).(*namespace.Namespace) - if !ok { - return nil, errors.New("nexus namespace not found in context") - } - return namespaceEntry, nil -} diff --git a/common/rpc/interceptor/nexus/nexus.go b/common/rpc/interceptor/nexus/nexus.go new file mode 100644 index 00000000000..a1f2f6c939e --- /dev/null +++ b/common/rpc/interceptor/nexus/nexus.go @@ -0,0 +1,227 @@ +package nexus + +import ( + "context" + "errors" + "net/http" + "slices" + + "github.com/nexus-rpc/sdk-go/nexus" + "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/nexus/nexusrpc" +) + +type HandlerFunc func(ctx context.Context, in InterceptorInput) (any, error) + +type Interceptor func(ctx context.Context, in InterceptorInput, next HandlerFunc) (any, error) + +type InterceptorInput interface { + ServiceName() string + OperationName() string + NamespaceName() string // TODO: this should just use NamespaceEntry() instead + ForwardingInfo() ForwardingInfo + APIName() string // analogous to the gRPC FullMethod + NamespaceEntry() (*namespace.Namespace, error) + EndpointName() string + sealNexusOp() +} + +var ( + _ InterceptorInput = StartOpInput{} + _ InterceptorInput = CancelOpInput{} + _ InterceptorInput = CompleteOpInput{} +) + +// ForwardingInfo contains the request data needed to forward a Nexus operation. +type ForwardingInfo struct { + OriginalRequestHeaders http.Header + TaskQueue string + EndpointID string + EndpointName string + BusinessID string +} + +type InterceptorError struct { + // wrapped error + Err error + // Outcome tag for metrics reporting, (draft-review: should Outcomes be enum or at least constants instead) + Outcome string +} + +func (t *InterceptorError) Error() string { + return t.Err.Error() +} + +func (t *InterceptorError) Unwrap() error { + return t.Err +} + +// RequestMetadata carries request metadata that is only known once the handler +// has resolved it (e.g. after a namespace registry lookup), and so cannot be supplied +// at InterceptorInput construction time. Set via nexusOpBase.WithRequestMetadata. +type RequestMetadata struct { + APIName string + NamespaceEntry *namespace.Namespace + EndpointName string +} + +// container for ServiceName(), OperationName(), NamespaceName(), ForwardingInfo(), and +// the fields in RequestMetadata. +type nexusOpBase struct { + serviceName, operation, namespaceName string + forwardingInfo ForwardingInfo + requestMetadata RequestMetadata +} + +func (b *nexusOpBase) WithForwardingInfo(info ForwardingInfo) { + b.forwardingInfo = info +} + +func (b *nexusOpBase) WithRequestMetadata(metadata RequestMetadata) { + b.requestMetadata = metadata +} + +func (b nexusOpBase) ServiceName() string { + return b.serviceName +} + +func (b nexusOpBase) OperationName() string { + return b.operation +} + +func (b nexusOpBase) NamespaceName() string { + return b.namespaceName +} + +func (b nexusOpBase) ForwardingInfo() ForwardingInfo { + return b.forwardingInfo +} + +func (b nexusOpBase) APIName() string { + return b.requestMetadata.APIName +} + +func (b nexusOpBase) NamespaceEntry() (*namespace.Namespace, error) { + if b.requestMetadata.NamespaceEntry == nil { + return nil, errors.New("namespace not found in request metadata") + } + return b.requestMetadata.NamespaceEntry, nil +} + +func (b nexusOpBase) EndpointName() string { + return b.requestMetadata.EndpointName +} + +func (nexusOpBase) sealNexusOp() {} + +func HeaderFromInterceptorInput(in InterceptorInput) (headers.HeaderGetter, error) { + switch opts := in.(type) { + case StartOpInput: + return opts.StartOperationOptions.Header, nil + case CancelOpInput: + return opts.CancelOperationOptions.Header, nil + case CompleteOpInput: + if opts.CompletionRequest == nil || opts.CompletionRequest.HTTPRequest == nil { + return nil, errors.New("nexus completion request not found") + } + return opts.CompletionRequest.HTTPRequest.Header, nil + default: + return nil, errors.New("unknown Nexus interceptor input") + } +} + +// draft-review: verify that these are the "right" methods/names +// TBD: is this different from api.MethodName(in.APIName())? +// +//nolint:staticcheck +func MethodName(in InterceptorInput) string { + switch in.(type) { + case StartOpInput: + return "StartNexusOperation" + case CancelOpInput: + return "CancelNexusOperation" + case CompleteOpInput: + return "CompleteNexusOperation" + default: + return "" + } +} + +type StartOpInput struct { + nexusOpBase + StartOperationOptions nexus.StartOperationOptions + StartOperationInput *nexus.LazyValue +} + +func NewStartOpInput( + serviceName string, + operation string, + namespaceName string, + options nexus.StartOperationOptions, + input *nexus.LazyValue, +) StartOpInput { + return StartOpInput{ + nexusOpBase: nexusOpBase{ + serviceName: serviceName, + operation: operation, + namespaceName: namespaceName, + }, + StartOperationOptions: options, + StartOperationInput: input, + } +} + +type CancelOpInput struct { + nexusOpBase + CancelOperationOptions nexus.CancelOperationOptions + CancellationToken string +} + +func NewCancelOpInput( + serviceName string, + operation string, + namespaceName string, + options nexus.CancelOperationOptions, + cancellationToken string, +) CancelOpInput { + return CancelOpInput{ + nexusOpBase: nexusOpBase{ + serviceName: serviceName, + operation: operation, + namespaceName: namespaceName, + }, + CancelOperationOptions: options, + CancellationToken: cancellationToken, + } +} + +type CompleteOpInput struct { + nexusOpBase + CompletionRequest *nexusrpc.CompletionRequest +} + +// draft-review: Complete doesnt need servicename/op - verify +// +//nolint:staticcheck +func NewCompleteOpInput( + namespaceName string, + request *nexusrpc.CompletionRequest, +) CompleteOpInput { + return CompleteOpInput{ + nexusOpBase: nexusOpBase{ + namespaceName: namespaceName, + }, + CompletionRequest: request, + } +} + +func ChainInterceptors(final HandlerFunc, chain []Interceptor) HandlerFunc { + for _, curr := range slices.Backward(chain) { + next := final + final = func(ctx context.Context, opts InterceptorInput) (any, error) { + return curr(ctx, opts, next) + } + } + return final +} diff --git a/common/rpc/interceptor/nexus_test.go b/common/rpc/interceptor/nexus/nexus_test.go similarity index 61% rename from common/rpc/interceptor/nexus_test.go rename to common/rpc/interceptor/nexus/nexus_test.go index 12f7d9526a8..8eb71159b97 100644 --- a/common/rpc/interceptor/nexus_test.go +++ b/common/rpc/interceptor/nexus/nexus_test.go @@ -1,4 +1,4 @@ -package interceptor +package nexus import ( "context" @@ -9,14 +9,14 @@ import ( func TestChainNexusInterceptors(t *testing.T) { var calls []string - chain := []NexusInterceptor{ - func(ctx context.Context, in NexusInterceptorInput, next NexusHandlerFunc) (any, error) { + chain := []Interceptor{ + func(ctx context.Context, in InterceptorInput, next HandlerFunc) (any, error) { calls = append(calls, "first-before") result, err := next(ctx, in) calls = append(calls, "first-after") return result, err }, - func(ctx context.Context, in NexusInterceptorInput, next NexusHandlerFunc) (any, error) { + func(ctx context.Context, in InterceptorInput, next HandlerFunc) (any, error) { calls = append(calls, "second-before") result, err := next(ctx, in) calls = append(calls, "second-after") @@ -24,10 +24,10 @@ func TestChainNexusInterceptors(t *testing.T) { }, } - result, err := ChainNexusInterceptors(func(context.Context, NexusInterceptorInput) (any, error) { + result, err := ChainInterceptors(func(context.Context, InterceptorInput) (any, error) { calls = append(calls, "handler") return "result", nil - }, chain)(context.Background(), StartNexusOpInput{}) + }, chain)(context.Background(), StartOpInput{}) require.NoError(t, err) require.Equal(t, "result", result) @@ -42,18 +42,18 @@ func TestChainNexusInterceptors(t *testing.T) { func TestChainNexusInterceptorsShortCircuit(t *testing.T) { var calls []string - chain := []NexusInterceptor{ - func(context.Context, NexusInterceptorInput, NexusHandlerFunc) (any, error) { + chain := []Interceptor{ + func(context.Context, InterceptorInput, HandlerFunc) (any, error) { calls = append(calls, "interceptor") // dont call next - just return return "intercepted", nil }, } - result, err := ChainNexusInterceptors(func(context.Context, NexusInterceptorInput) (any, error) { + result, err := ChainInterceptors(func(context.Context, InterceptorInput) (any, error) { calls = append(calls, "handler") return "handler", nil - }, chain)(context.Background(), StartNexusOpInput{}) + }, chain)(context.Background(), StartOpInput{}) require.NoError(t, err) require.Equal(t, "intercepted", result) diff --git a/common/rpc/interceptor/rate_limit.go b/common/rpc/interceptor/rate_limit.go index 785ff499b97..513f5c9ed1c 100644 --- a/common/rpc/interceptor/rate_limit.go +++ b/common/rpc/interceptor/rate_limit.go @@ -10,6 +10,7 @@ import ( "go.temporal.io/server/common/headers" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/quotas" + "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -95,25 +96,18 @@ func (i *RateLimitInterceptor) Allow( // InterceptNexus enforces the global rate limit for a Nexus request. func (i *RateLimitInterceptor) InterceptNexus( ctx context.Context, - in NexusInterceptorInput, - next NexusHandlerFunc, + in nexus.InterceptorInput, + next nexus.HandlerFunc, ) (any, error) { - apiName, err := NexusAPINameFromContext(ctx) + header, err := nexus.HeaderFromInterceptorInput(in) if err != nil { - return nil, &InterceptorError{ + return nil, &nexus.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, true), Outcome: "interceptor_failed", } } - header, err := NexusHeaderFromInterceptorInput(in) - if err != nil { - return nil, &InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), - Outcome: "interceptor_failed", - } - } - if err := i.Allow(apiName, header); err != nil { - return nil, &InterceptorError{ + if err := i.Allow(in.APIName(), header); err != nil { + return nil, &nexus.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, true), Outcome: "global_rate_limited", } diff --git a/common/rpc/interceptor/rate_limit_test.go b/common/rpc/interceptor/rate_limit_test.go index db04f608644..445dff1a9ed 100644 --- a/common/rpc/interceptor/rate_limit_test.go +++ b/common/rpc/interceptor/rate_limit_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.temporal.io/server/common/quotas" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.uber.org/mock/gomock" "google.golang.org/grpc" ) @@ -31,40 +32,36 @@ func (s *rateLimitInterceptorSuite) TestInterceptNexus() { for _, tc := range []struct { name string apiName string - input NexusInterceptorInput + input interceptornexus.InterceptorInput allow *bool nextCalled bool expectedOutcome string }{ - {name: "allowed", apiName: "NexusOperation", input: NewStartNexusOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil), allow: new(true), nextCalled: true}, - {name: "rate limited", apiName: "NexusOperation", input: NewStartNexusOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil), allow: new(false), expectedOutcome: "global_rate_limited"}, - {name: "missing API name", expectedOutcome: "interceptor_failed"}, - {name: "missing request header", apiName: "NexusOperation", input: NewCompleteNexusOpInput(testNamespace, nil), expectedOutcome: "interceptor_failed"}, + {name: "allowed", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil), "NexusOperation"), allow: new(true), nextCalled: true}, + {name: "rate limited", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil), "NexusOperation"), allow: new(false), expectedOutcome: "global_rate_limited"}, + {name: "missing request header", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewCompleteOpInput(testNamespace, nil), "NexusOperation"), expectedOutcome: "interceptor_failed"}, } { s.Run(tc.name, func() { ctx := context.Background() interceptor := NewRateLimitInterceptor(s.mockRateLimiter, nil) - if tc.apiName != "" { - ctx = WithNexusAPIName(ctx, tc.apiName) - } if tc.allow != nil { s.mockRateLimiter.EXPECT().Allow(gomock.Any(), gomock.Any()).Return(*tc.allow) } input := tc.input if input == nil { - input = NewStartNexusOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil) + input = interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil) } nextCalled := false _, err := interceptor.InterceptNexus( ctx, input, - func(context.Context, NexusInterceptorInput) (any, error) { + func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil }, ) if tc.expectedOutcome != "" { - var interceptorErr *InterceptorError + var interceptorErr *interceptornexus.InterceptorError s.ErrorAs(err, &interceptorErr) s.Equal(tc.expectedOutcome, interceptorErr.Outcome) } else { diff --git a/common/rpc/interceptor/retry.go b/common/rpc/interceptor/retry.go index fe2f818b893..7cd6d7724b8 100644 --- a/common/rpc/interceptor/retry.go +++ b/common/rpc/interceptor/retry.go @@ -4,6 +4,7 @@ import ( "context" "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -42,3 +43,14 @@ func (i *RetryableInterceptor) Intercept( err := backoff.ThrottleRetryContext(ctx, op, i.policy, i.isRetryable) return response, err } + +// TBD: evaluate if adding retry is necessary/correct for Nexus +// +//nolint:staticcheck +func (i *RetryableInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + return next(ctx, in) +} diff --git a/common/rpc/interceptor/routing_key_interceptor.go b/common/rpc/interceptor/routing_key_interceptor.go index 82b73ce6e83..e1962ce0e21 100644 --- a/common/rpc/interceptor/routing_key_interceptor.go +++ b/common/rpc/interceptor/routing_key_interceptor.go @@ -6,6 +6,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -121,6 +122,23 @@ func (i *RoutingKeyInterceptor) Intercept( return handler(ctx, req) } +// TBD: check if this is needed or if this can also be a passthrough +// +//nolint:staticcheck +func (i *RoutingKeyInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + if in.ForwardingInfo().BusinessID != "" { + key := namespace.RoutingKey{ + ID: in.ForwardingInfo().BusinessID, + } + ctx = AddRoutingKeyToContext(ctx, key) + } + return next(ctx, in) +} + // AddRoutingKeyToContext adds the routing Key to the context func AddRoutingKeyToContext(ctx context.Context, routingKey namespace.RoutingKey) context.Context { return context.WithValue(ctx, routingKeyCtxKey, routingKey) diff --git a/common/rpc/interceptor/sdk_version.go b/common/rpc/interceptor/sdk_version.go index 7106fcc2b3a..0f5ac1c3196 100644 --- a/common/rpc/interceptor/sdk_version.go +++ b/common/rpc/interceptor/sdk_version.go @@ -6,6 +6,7 @@ import ( "go.temporal.io/server/common/headers" commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/versioninfo" "google.golang.org/grpc" ) @@ -48,8 +49,8 @@ func (vi *SDKVersionInterceptor) Intercept( // InterceptNexus records and validates the SDK version for a Nexus request. func (vi *SDKVersionInterceptor) InterceptNexus( ctx context.Context, - in NexusInterceptorInput, - next NexusHandlerFunc, + in nexus.InterceptorInput, + next nexus.HandlerFunc, ) (any, error) { // draft-review: RecordSDKInfo didnt exist before, nice to add sdkName, sdkVersion := headers.GetClientNameAndVersion(ctx) @@ -57,7 +58,7 @@ func (vi *SDKVersionInterceptor) InterceptNexus( vi.RecordSDKInfo(sdkName, sdkVersion) } if err := vi.versionChecker.ClientSupported(ctx); err != nil { - return nil, &InterceptorError{ + return nil, &nexus.InterceptorError{ Err: commonnexus.ConvertGRPCError(err, true), Outcome: "unsupported_client", } diff --git a/common/rpc/interceptor/sdk_version_test.go b/common/rpc/interceptor/sdk_version_test.go index 2376ba6a97b..81fb3eb783c 100644 --- a/common/rpc/interceptor/sdk_version_test.go +++ b/common/rpc/interceptor/sdk_version_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.temporal.io/server/common/headers" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/versioninfo" ) @@ -101,14 +102,14 @@ func TestSDKVersionInterceptNexus(t *testing.T) { nextCalled := false _, err := interceptor.InterceptNexus( tc.ctx, - NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), - func(context.Context, NexusInterceptorInput) (any, error) { + interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil }, ) if tc.expectedOutcome != "" { - var interceptorErr *InterceptorError + var interceptorErr *interceptornexus.InterceptorError require.ErrorAs(t, err, &interceptorErr) require.Equal(t, tc.expectedOutcome, interceptorErr.Outcome) require.False(t, nextCalled) diff --git a/common/rpc/interceptor/service_error_interceptor.go b/common/rpc/interceptor/service_error_interceptor.go index 67abfc827fd..41eb29d84c4 100644 --- a/common/rpc/interceptor/service_error_interceptor.go +++ b/common/rpc/interceptor/service_error_interceptor.go @@ -9,6 +9,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/persistence/serialization" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/util" "google.golang.org/grpc" "google.golang.org/grpc/status" @@ -44,6 +45,27 @@ func (i *ServiceErrorInterceptor) Intercept( ) (any, error) { resp, err := i.capturePanicHandler(ctx, req, handler) + return resp, i.transformError(err) +} + +// InterceptNexus is a no-op: unlike the gRPC path, every error reaching the Nexus +// chain is already converted to a *nexus.HandlerError/*nexus.OperationError at its +// origin (see commonnexus.ConvertGRPCError call sites in nexus_handler.go and the +// other Nexus interceptors), so there's nothing left for transformError to do. This +// method exists only so ServiceErrorInterceptor keeps its chain position for parity +// with the gRPC ordering. +func (i *ServiceErrorInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + return next(ctx, in) +} + +func (i *ServiceErrorInterceptor) transformError(err error) error { + if err == nil { + return nil + } var deserializationError *serialization.DeserializationError var serializationError *serialization.SerializationError // convert serialization errors to be captured as serviceerrors across gRPC calls @@ -59,8 +81,7 @@ func (i *ServiceErrorInterceptor) Intercept( p.Message = util.TruncateUTF8(p.Message, maxLength-len(truncatedSuffix)) + truncatedSuffix st = status.FromProto(p) } - - return resp, st.Err() + return st.Err() } func (i *ServiceErrorInterceptor) capturePanicHandler( diff --git a/common/rpc/interceptor/slow_request_logger.go b/common/rpc/interceptor/slow_request_logger.go index dd8dddc7871..d3d0c617463 100644 --- a/common/rpc/interceptor/slow_request_logger.go +++ b/common/rpc/interceptor/slow_request_logger.go @@ -9,6 +9,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/rpc/interceptor/logtags" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/tasktoken" "google.golang.org/grpc" ) @@ -36,27 +37,45 @@ func (i *SlowRequestLoggerInterceptor) Intercept( info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ) (any, error) { - // Long-polled methods aren't useful logged. - if api.GetMethodMetadata(info.FullMethod).Polling == api.PollingNone { - startTime := time.Now() + tracker := i.trackSlowRequestFn(info.FullMethod, request) + defer tracker() + + return handler(ctx, request) +} - defer func() { - elapsed := time.Since(startTime) - if elapsed > i.slowRequestThreshold() { - i.logSlowRequest(request, info, elapsed) - } - }() +func (i *SlowRequestLoggerInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + tracker := i.trackSlowRequestFn(in.OperationName(), in) + defer tracker() + return next(ctx, in) +} + +func (i *SlowRequestLoggerInterceptor) trackSlowRequestFn(operationName string, req any) func() { + // Long-polled methods aren't useful logged. + // If it's a polled method, return a no-op function to defer + if api.GetMethodMetadata(operationName).Polling != api.PollingNone { + return func() {} } - return handler(ctx, request) + startTime := time.Now() + + // Return the cleanup closure for the parent to defer + return func() { + elapsed := time.Since(startTime) + if elapsed > i.slowRequestThreshold() { + i.logSlowRequest(req, operationName, elapsed) + } + } } func (i *SlowRequestLoggerInterceptor) logSlowRequest( request any, - info *grpc.UnaryServerInfo, + method string, elapsed time.Duration, ) { - method := info.FullMethod tags := i.workflowTags.Extract(request, method) tags = append(tags, tag.Duration("duration", elapsed)) diff --git a/common/rpc/interceptor/telemetry.go b/common/rpc/interceptor/telemetry.go index ae110e9a8b3..ddbd1fd310c 100644 --- a/common/rpc/interceptor/telemetry.go +++ b/common/rpc/interceptor/telemetry.go @@ -19,6 +19,7 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/rpc/interceptor/logtags" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/service/frontend/configs" "google.golang.org/grpc" @@ -47,6 +48,7 @@ type ( // SetFailureSource records which side produced a failure. Only the start/cancel // handlers return this to the caller; the completion handler discards it. SetFailureSource(string) + // TBD - check if this is really needed, could remove the error handler interceptor // HandleRequestError reports a failed request to the shared ErrorHandler. HandleRequestError(error) } @@ -64,7 +66,7 @@ var ( updateResponseMessageBody anypb.Any _ = updateResponseMessageBody.MarshalFrom(&updatepb.Response{}) - _ grpc.UnaryServerInterceptor = (*TelemetryInterceptor)(nil).UnaryIntercept + _ grpc.UnaryServerInterceptor = (*TelemetryInterceptor)(nil).Intercept _ grpc.StreamServerInterceptor = (*TelemetryInterceptor)(nil).StreamIntercept ) @@ -179,7 +181,7 @@ func telemetryOverrideOperationTag(fullName, operation string) string { return operation } -func (ti *TelemetryInterceptor) UnaryIntercept( +func (ti *TelemetryInterceptor) Intercept( ctx context.Context, req any, info *grpc.UnaryServerInfo, @@ -243,8 +245,8 @@ func TelemetryContextFromContext(ctx context.Context) (TelemetryContext, error) // GetMetricsHandlerFromContext. func (ti *TelemetryInterceptor) InterceptNexus( ctx context.Context, - in NexusInterceptorInput, - next NexusHandlerFunc, + in nexus.InterceptorInput, + next nexus.HandlerFunc, ) (out any, retErr error) { // draft-review: it it not worth splitting the metrics into pre and post forwarder interceptor groups. @@ -269,13 +271,13 @@ func (ti *TelemetryInterceptor) InterceptNexus( defer func() { reportErr := retErr - if taggedErr, ok := errors.AsType[*InterceptorError](retErr); ok { + if taggedErr, ok := errors.AsType[*nexus.InterceptorError](retErr); ok { telemetryContext.SetMetricsOutcome(taggedErr.Outcome) reportErr = taggedErr.Err } metricsHandler := telemetryContext.MetricsHandler(reportErr) switch in.(type) { - case CompleteNexusOpInput: + case nexus.CompleteOpInput: metricsHandler.Counter(metrics.NexusCompletionRequests.Name()).Record(1) metricsHandler.Histogram(metrics.NexusCompletionLatencyHistogram.Name(), metrics.Milliseconds).Record(time.Since(startTime).Milliseconds()) default: diff --git a/common/rpc/interceptor/telemetry_test.go b/common/rpc/interceptor/telemetry_test.go index 6b15046e1db..042f88b608e 100644 --- a/common/rpc/interceptor/telemetry_test.go +++ b/common/rpc/interceptor/telemetry_test.go @@ -23,6 +23,7 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" serviceerrors "go.temporal.io/server/common/serviceerror" "go.uber.org/mock/gomock" "google.golang.org/grpc/codes" @@ -59,11 +60,11 @@ func (c *nexusTelemetryContext) HandleRequestError(err error) { } func TestTelemetryInterceptNexus(t *testing.T) { - input := NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) + input := interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) for _, tc := range []struct { name string setContext bool - handler NexusHandlerFunc + handler interceptornexus.HandlerFunc expectedOutcome string expectedError error nextCalled bool @@ -72,7 +73,7 @@ func TestTelemetryInterceptNexus(t *testing.T) { { name: "regular telemetry capture", setContext: true, - handler: func(context.Context, NexusInterceptorInput) (any, error) { + handler: func(context.Context, interceptornexus.InterceptorInput) (any, error) { return nil, nil }, nextCalled: true, @@ -80,7 +81,7 @@ func TestTelemetryInterceptNexus(t *testing.T) { }, { name: "missing telemetry context", - handler: func(context.Context, NexusInterceptorInput) (any, error) { + handler: func(context.Context, interceptornexus.InterceptorInput) (any, error) { return nil, nil }, expectedError: errors.New("telemetry context not found"), @@ -88,18 +89,18 @@ func TestTelemetryInterceptNexus(t *testing.T) { { name: "tagged error", setContext: true, - handler: func(context.Context, NexusInterceptorInput) (any, error) { - return nil, &InterceptorError{Err: errors.New("rejected"), Outcome: "rejected"} + handler: func(context.Context, interceptornexus.InterceptorInput) (any, error) { + return nil, &interceptornexus.InterceptorError{Err: errors.New("rejected"), Outcome: "rejected"} }, expectedOutcome: "rejected", - expectedError: &InterceptorError{Err: errors.New("rejected"), Outcome: "rejected"}, + expectedError: &interceptornexus.InterceptorError{Err: errors.New("rejected"), Outcome: "rejected"}, nextCalled: true, expectHandled: true, }, { name: "ensure metrics still captured on panics", setContext: true, - handler: func(context.Context, NexusInterceptorInput) (any, error) { + handler: func(context.Context, interceptornexus.InterceptorInput) (any, error) { panic("") }, expectedError: errors.New("panic: "), @@ -114,7 +115,7 @@ func TestTelemetryInterceptNexus(t *testing.T) { ctx = WithTelemetryContext(ctx, telemetryContext) } nextCalled := false - _, err := (&TelemetryInterceptor{}).InterceptNexus(ctx, input, func(ctx context.Context, input NexusInterceptorInput) (any, error) { + _, err := (&TelemetryInterceptor{}).InterceptNexus(ctx, input, func(ctx context.Context, input interceptornexus.InterceptorInput) (any, error) { nextCalled = true return tc.handler(ctx, input) }) diff --git a/service/frontend/frontend_interceptors.go b/service/frontend/frontend_interceptors.go new file mode 100644 index 00000000000..01aab2ec2a9 --- /dev/null +++ b/service/frontend/frontend_interceptors.go @@ -0,0 +1,207 @@ +package frontend + +import ( + "context" + + "go.temporal.io/server/chasm" + "go.temporal.io/server/common/authorization" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/rpc/grpcfaults" + "go.temporal.io/server/common/rpc/interceptor" + "go.temporal.io/server/common/rpc/interceptor/nexus" + "go.temporal.io/server/common/testing/grpcfaultstest" + "go.temporal.io/server/common/testing/testhooks" + "google.golang.org/grpc" +) + +// Interceptor is a unified interface for gRPC and Nexus interceptors +type Interceptor interface { + // gRPC Interceptor + Intercept( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, + ) (any, error) + // Nexus Interceptor + InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, + ) (any, error) +} + +type InterceptorsProvider struct { + interceptors []Interceptor + // nexusInterceptors are different from interceptors due to a few reasons + // eventual goal is to have exact same set and order + ninterceptors []Interceptor + retryableInterceptor *interceptor.RetryableInterceptor // required to be last in chain after custom interceptors + customGRPCInterceptors []grpc.UnaryServerInterceptor // required for legacy reasons + faultGenerator grpcfaults.Generator +} + +func NewInterceptorsProvider( + maskInternalErrorDetailsInterceptor *interceptor.MaskInternalErrorDetailsInterceptor, + serviceErrorInterceptor *interceptor.ServiceErrorInterceptor, + frontendServiceErrorInterceptor *interceptor.FrontendServiceErrorInterceptor, + businessIDInterceptor *interceptor.RoutingKeyInterceptor, + namespaceValidatorInterceptor *interceptor.NamespaceValidatorInterceptor, + namespaceLogInterceptor *interceptor.NamespaceLogInterceptor, + metricsCtxInjectorInterceptor *metricsCtxInjectorInterceptor, + authInterceptor *authorization.Interceptor, + namespaceHandoverInterceptor *interceptor.NamespaceHandoverInterceptor, + redirectionSlot *redirectionWrapper, + telemetryInterceptor *interceptor.TelemetryInterceptor, + healthInterceptor *interceptor.HealthInterceptor, + namespaceStateValidatorInterceptor *interceptor.NamespaceStateValidatorInterceptor, + namespaceCountLimiterInterceptor *interceptor.ConcurrentRequestLimitInterceptor, + namespaceRateLimiterInterceptorWrapper *interceptor.NamespaceRateLimitInterceptorWrapper, + retryableInterceptor *interceptor.RetryableInterceptor, + rateLimitInterceptor *interceptor.RateLimitInterceptor, + sdkVersionInterceptor *interceptor.SDKVersionInterceptor, + callerInfoInterceptor *interceptor.CallerInfoInterceptor, + slowRequestLoggerInterceptor *interceptor.SlowRequestLoggerInterceptor, + chasmRequestVisibilityInterceptor *chasm.ChasmVisibilityInterceptor, + contextMetadataInterceptor *interceptor.ContextMetadataInterceptor, + customGRPCInterceptors []grpc.UnaryServerInterceptor, + customInterceptors []Interceptor, + testHooks testhooks.TestHooks, +) *InterceptorsProvider { + + interceptors := []Interceptor{ + maskInternalErrorDetailsInterceptor, + serviceErrorInterceptor, + frontendServiceErrorInterceptor, + businessIDInterceptor, + namespaceStateValidatorInterceptor, + namespaceLogInterceptor, + metricsCtxInjectorInterceptor, + // for gRPC, auth is before telemetry + authInterceptor, + namespaceHandoverInterceptor, + redirectionSlot, + telemetryInterceptor, + // rest of the chain is identical for gRPC and Nexus + healthInterceptor, + namespaceValidatorInterceptor, + namespaceCountLimiterInterceptor, + namespaceRateLimiterInterceptorWrapper, + rateLimitInterceptor, + sdkVersionInterceptor, + callerInfoInterceptor, + slowRequestLoggerInterceptor, + chasmRequestVisibilityInterceptor, + contextMetadataInterceptor, + } + ninterceptors := []Interceptor{ + maskInternalErrorDetailsInterceptor, + serviceErrorInterceptor, + frontendServiceErrorInterceptor, + businessIDInterceptor, + namespaceStateValidatorInterceptor, + namespaceLogInterceptor, + metricsCtxInjectorInterceptor, + // for Nexus, telemetry is before auth + telemetryInterceptor, + authInterceptor, + namespaceHandoverInterceptor, + redirectionSlot, + // rest of the chain is identical for gRPC and Nexus + healthInterceptor, + namespaceValidatorInterceptor, + namespaceCountLimiterInterceptor, + namespaceRateLimiterInterceptorWrapper, + rateLimitInterceptor, + sdkVersionInterceptor, + callerInfoInterceptor, + slowRequestLoggerInterceptor, + chasmRequestVisibilityInterceptor, + contextMetadataInterceptor, + } + // it is debatable if this should be *after* customGRPCInterceptors that are + // in use today. We will opt for this instead because relative ordering remains + // unchanged and anyone using customInterceptors should deprecate customGRPCInterceptors entirely + interceptors = append(interceptors, customInterceptors...) + + return &InterceptorsProvider{ + interceptors: interceptors, + ninterceptors: ninterceptors, + customGRPCInterceptors: customGRPCInterceptors, + retryableInterceptor: retryableInterceptor, + faultGenerator: grpcfaultstest.NewGenerator(testHooks), + } +} + +func (n *InterceptorsProvider) GetInterceptors() []grpc.UnaryServerInterceptor { + grpcInterceptors := []grpc.UnaryServerInterceptor{} + for _, i := range n.interceptors { + grpcInterceptors = append(grpcInterceptors, i.Intercept) + } + // custom interceptors chain after system interceptors + grpcInterceptors = append(grpcInterceptors, n.customGRPCInterceptors...) + grpcInterceptors = append(grpcInterceptors, n.retryableInterceptor.Intercept) + + if faultInterceptor := grpcfaults.UnaryServerInterceptor(n.faultGenerator); faultInterceptor != nil { + grpcInterceptors = append(grpcInterceptors, faultInterceptor) + } + return grpcInterceptors +} + +func (n *InterceptorsProvider) GetNexusInterceptors() []nexus.Interceptor { + nexusInterceptors := []nexus.Interceptor{} + for _, i := range n.ninterceptors { + nexusInterceptors = append(nexusInterceptors, i.InterceptNexus) + } + + nexusInterceptors = append(nexusInterceptors, n.retryableInterceptor.InterceptNexus) + return nexusInterceptors +} + +// redirectionWrapper is one chain position for both transports: gRPC DC redirection +// and Nexus HTTP forwarding. The implementations stay separate but are wrapped together +// for canonical ordering of interceptors for both gRPC and Nexus +type redirectionWrapper struct { + grpc *interceptor.Redirection + nexus *nexusForwardingInterceptor +} + +func (s *redirectionWrapper) Intercept( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, +) (any, error) { + return s.grpc.Intercept(ctx, req, info, handler) +} + +func (s *redirectionWrapper) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + return s.nexus.InterceptNexus(ctx, in, next) +} + +// tiny wrapper to inject metrics context and avoid +// cyclical dependencies in metrics/interceptors packages +type metricsCtxInjectorInterceptor struct{} + +func (m *metricsCtxInjectorInterceptor) Intercept( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, +) (any, error) { + ctxWithMetricsBaggage := metrics.AddMetricsContext(ctx) + return handler(ctxWithMetricsBaggage, req) +} + +func (m *metricsCtxInjectorInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + ctxWithMetricsBaggage := metrics.AddMetricsContext(ctx) + return next(ctxWithMetricsBaggage, in) +} diff --git a/service/frontend/fx.go b/service/frontend/fx.go index dfa937990f8..65c748be55a 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -43,12 +43,10 @@ import ( "go.temporal.io/server/common/resource" "go.temporal.io/server/common/rpc" "go.temporal.io/server/common/rpc/encryption" - "go.temporal.io/server/common/rpc/grpcfaults" "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/common/sdk" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/telemetry" - "go.temporal.io/server/common/testing/grpcfaultstest" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service" "go.temporal.io/server/service/frontend/configs" @@ -98,6 +96,8 @@ var Module = fx.Options( fx.Provide(interceptor.NewRoutingKeyExtractor), fx.Provide(BusinessIDInterceptorProvider), fx.Provide(RedirectionInterceptorProvider), + fx.Provide(RedirectionSlotProvider), + fx.Provide(NewMetricsContextInjectorInterceptor), fx.Provide(ErrorHandlerProvider), fx.Provide(TelemetryInterceptorProvider), fx.Provide(RetryableInterceptorProvider), @@ -105,6 +105,7 @@ var Module = fx.Options( fx.Provide(interceptor.NewHealthInterceptor), fx.Provide(NamespaceCountLimitInterceptorProvider), fx.Provide(NamespaceValidatorInterceptorProvider), + fx.Provide(NamespaceStateValidatorInterceptorProvider), fx.Provide(NamespaceRateLimitersProvider), fx.Provide(NamespaceRateLimitInterceptorProvider), fx.Provide(SDKVersionInterceptorProvider), @@ -126,12 +127,15 @@ var Module = fx.Options( fx.Provide(callbackValidatorProvider), fx.Provide(HandlerProvider), fx.Provide(AdminHandlerProvider), + fx.Provide(FrontendServiceErrorInterceptorProvider), fx.Provide(NamespaceDLQHandlerProvider), fx.Provide(OperatorHandlerProvider), fx.Provide(NewVersionChecker), fx.Provide(ServiceResolverProvider), fx.Provide(newNexusForwardingInterceptor), - fx.Provide(interceptor.NewNexusNamespaceRateLimitInterceptor), + fx.Provide(interceptor.NewNamespaceRateLimitInterceptorWrapper), + fx.Provide(NewInterceptorsProvider), + fx.Supply([]Interceptor(nil)), // placeholder for custom unified interceptors thaw will get chained fx.Provide(newNexusCompletionHandler), fx.Provide(NewNexusOperationHTTPHandler), fx.Provide(newNexusCompletionHTTPHandler), @@ -244,7 +248,10 @@ func GrpcServerOptionsProvider( namespaceRateLimiterInterceptor interceptor.NamespaceRateLimitInterceptor, namespaceCountLimiterInterceptor *interceptor.ConcurrentRequestLimitInterceptor, namespaceValidatorInterceptor *interceptor.NamespaceValidatorInterceptor, + namespaceStateValidatorInterceptor *interceptor.NamespaceStateValidatorInterceptor, + frontendServiceErrorInterceptor *interceptor.FrontendServiceErrorInterceptor, namespaceHandoverInterceptor *interceptor.NamespaceHandoverInterceptor, + interceptorsProvider *InterceptorsProvider, businessIDInterceptor *interceptor.RoutingKeyInterceptor, redirectionInterceptor *interceptor.Redirection, telemetryInterceptor *interceptor.TelemetryInterceptor, @@ -289,46 +296,45 @@ func GrpcServerOptionsProvider( if err != nil { logger.Fatal("creating gRPC server options failed", tag.Error(err)) } - unaryInterceptors := []grpc.UnaryServerInterceptor{ - // Order of interceptors is important - // Mask error interceptor should be the most outer interceptor since it handle the errors format - // Service Error Interceptor should be the next most outer interceptor on error handling - maskInternalErrorDetailsInterceptor.Intercept, - serviceErrorInterceptor.Intercept, - interceptor.NewFrontendServiceErrorInterceptor(logger), - // BusinessID interceptor extracts business ID and adds it to context for use, must be before any interceptor that touches namespaces (namespaceValidator, handoverInterceptor) - businessIDInterceptor.Intercept, - namespaceValidatorInterceptor.NamespaceValidateIntercept, - namespaceLogInterceptor.Intercept, // TODO: Deprecate this with a outer custom interceptor - metrics.NewServerMetricsContextInjectorInterceptor(), - authInterceptor.Intercept, - // Handover interceptor has to above redirection because the request will route to the correct cluster after handover completed. - // And retry cannot be performed before customInterceptors. - namespaceHandoverInterceptor.Intercept, - redirectionInterceptor.Intercept, - // Telemetry interceptor must be after redirection to ensure metrics are recorded in the correct cluster - telemetryInterceptor.UnaryIntercept, - healthInterceptor.Intercept, - namespaceValidatorInterceptor.StateValidationIntercept, - namespaceCountLimiterInterceptor.Intercept, - namespaceRateLimiterInterceptor.Intercept, - rateLimitInterceptor.Intercept, - sdkVersionInterceptor.Intercept, - callerInfoInterceptor.Intercept, - slowRequestLoggerInterceptor.Intercept, - chasmRequestVisibilityInterceptor.Intercept, - contextMetadataInterceptor.Intercept, - } - if len(customInterceptors) > 0 { - // TODO: Deprecate WithChainedFrontendGrpcInterceptors and provide a inner custom interceptor - unaryInterceptors = append(unaryInterceptors, customInterceptors...) - } - faultGenerator := grpcfaultstest.NewGenerator(testHooks) - if faultInterceptor := grpcfaults.UnaryServerInterceptor(faultGenerator); faultInterceptor != nil { - unaryInterceptors = append(unaryInterceptors, faultInterceptor) - } - // retry interceptor should be the most inner interceptor - unaryInterceptors = append(unaryInterceptors, retryableInterceptor.Intercept) + // unaryInterceptors := []grpc.UnaryServerInterceptor{ + // // Order of interceptors is important + // // Mask error interceptor should be the most outer interceptor since it handle the errors format + // // Service Error Interceptor should be the next most outer interceptor on error handling + // maskInternalErrorDetailsInterceptor.Intercept, + // serviceErrorInterceptor.Intercept, + // frontendServiceErrorInterceptor.Intercept, + // //interceptor.NewFrontendServiceErrorInterceptor(logger), + // // BusinessID interceptor extracts business ID and adds it to context for use, must be before any interceptor that touches namespaces (namespaceValidator, handoverInterceptor) + // businessIDInterceptor.Intercept, + // namespaceStateValidatorInterceptor.Intercept, + // namespaceLogInterceptor.Intercept, // TODO: Deprecate this with a outer custom interceptor + // metrics.NewServerMetricsContextInjectorInterceptor(), // TODO + // authInterceptor.Intercept, + // // Handover interceptor has to above redirection because the request will route to the correct cluster after handover completed. + // // And retry cannot be performed before customInterceptors. + // namespaceHandoverInterceptor.Intercept, + // redirectionInterceptor.Intercept, // TODO, this will have to merge with the nexus frontend interceptor, eval later + // // Telemetry interceptor must be after redirection to ensure metrics are recorded in the correct cluster + // telemetryInterceptor.Intercept, + // healthInterceptor.Intercept, + // namespaceValidatorInterceptor.Intercept, + // namespaceCountLimiterInterceptor.Intercept, + // namespaceRateLimiterInterceptor.Intercept, + // rateLimitInterceptor.Intercept, + // sdkVersionInterceptor.Intercept, + // callerInfoInterceptor.Intercept, + // slowRequestLoggerInterceptor.Intercept, + // chasmRequestVisibilityInterceptor.Intercept, //TODO: this will require nexus interceptor types to be moved out + // contextMetadataInterceptor.Intercept, + // } + // if len(customInterceptors) > 0 { + // // TODO: Deprecate WithChainedFrontendGrpcInterceptors and provide a inner custom interceptor + // unaryInterceptors = append(unaryInterceptors, customInterceptors...) + // } + // // retry interceptor should be the most inner interceptor + // unaryInterceptors = append(unaryInterceptors, retryableInterceptor.Intercept) + + unaryInterceptors := interceptorsProvider.GetInterceptors() streamInterceptor := []grpc.StreamServerInterceptor{ authInterceptor.InterceptStream, @@ -401,6 +407,20 @@ func RetryableInterceptorProvider() *interceptor.RetryableInterceptor { ) } +func RedirectionSlotProvider( + redirectionInterceptor *interceptor.Redirection, + nexusForwarder *nexusForwardingInterceptor, +) *redirectionWrapper { + return &redirectionWrapper{ + grpc: redirectionInterceptor, + nexus: nexusForwarder, + } +} + +func NewMetricsContextInjectorInterceptor() *metricsCtxInjectorInterceptor { + return &metricsCtxInjectorInterceptor{} +} + func RedirectionInterceptorProvider( configuration *Config, namespaceCache namespace.Registry, @@ -694,6 +714,12 @@ func NamespaceValidatorInterceptorProvider( ) } +func NamespaceStateValidatorInterceptorProvider( + nvi *interceptor.NamespaceValidatorInterceptor, +) *interceptor.NamespaceStateValidatorInterceptor { + return interceptor.NewNamespaceStateValidatorInterceptor(nvi) +} + func SDKVersionInterceptorProvider() *interceptor.SDKVersionInterceptor { return interceptor.NewSDKVersionInterceptor() } @@ -714,6 +740,12 @@ func SlowRequestLoggerInterceptorProvider( ) } +func FrontendServiceErrorInterceptorProvider( + logger log.Logger, +) *interceptor.FrontendServiceErrorInterceptor { + return interceptor.NewFrontendServiceErrorInterceptorWrapper(logger) +} + func PersistenceRateLimitingParamsProvider( serviceConfig *Config, persistenceLazyLoadedServiceResolver service.PersistenceLazyLoadedServiceResolver, diff --git a/service/frontend/fx_test.go b/service/frontend/fx_test.go index 39a7ea121d3..e04c3196089 100644 --- a/service/frontend/fx_test.go +++ b/service/frontend/fx_test.go @@ -235,7 +235,7 @@ func TestRateLimitInterceptorProvider(t *testing.T) { svc := &testSvc{} server := grpc.NewServer(grpc.ChainUnaryInterceptor( serviceErrorInterceptor.Intercept, - interceptor.NewFrontendServiceErrorInterceptor(log.NewTestLogger()), + interceptor.NewFrontendServiceErrorInterceptorWrapper(log.NewTestLogger()).Intercept, rateLimitInterceptor.Intercept, )) workflowservice.RegisterWorkflowServiceServer(server, svc) @@ -603,7 +603,7 @@ func TestNamespaceRateLimitInterceptorProvider(t *testing.T) { svc := &testSvc{} server := grpc.NewServer(grpc.ChainUnaryInterceptor( serviceErrorInterceptor.Intercept, - interceptor.NewFrontendServiceErrorInterceptor(log.NewTestLogger()), + interceptor.NewFrontendServiceErrorInterceptorWrapper(log.NewTestLogger()).Intercept, rateLimitInterceptor.Intercept, )) workflowservice.RegisterWorkflowServiceServer(server, svc) @@ -798,7 +798,7 @@ func TestNamespaceRateLimitMetrics(t *testing.T) { svc := &testSvc{} server := grpc.NewServer(grpc.ChainUnaryInterceptor( serviceErrorInterceptor.Intercept, - interceptor.NewFrontendServiceErrorInterceptor(log.NewTestLogger()), + interceptor.NewFrontendServiceErrorInterceptorWrapper(log.NewTestLogger()).Intercept, rateLimitInterceptor.Intercept, )) workflowservice.RegisterWorkflowServiceServer(server, svc) diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index cbc8b89356e..eb5355b4512 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -26,6 +26,7 @@ import ( "go.temporal.io/server/common/resource" "go.temporal.io/server/common/rpc" "go.temporal.io/server/common/rpc/interceptor" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/nexusworkflowref" "go.temporal.io/server/service/frontend/configs" "go.temporal.io/server/service/history/consts" @@ -38,25 +39,17 @@ const nexusCompletionAPIName = configs.CompleteNexusOperation const nexusCompletionMethodName = "CompleteNexusOperation" type nexusCompletionHandler struct { - ClusterMetadata cluster.Metadata - NamespaceRegistry namespace.Registry - Logger log.Logger - MetricsHandler metrics.Handler - Config *Config - CallbackTokenGenerator *commonnexus.CallbackTokenGenerator - HistoryClient resource.HistoryClient - // TelemetryInterceptor *interceptor.TelemetryInterceptor - RequestErrorHandler *interceptor.RequestErrorHandler - // NamespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor - // NamespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor - // NamespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor - // RateLimitInterceptor *interceptor.RateLimitInterceptor - AuthInterceptor *authorization.Interceptor - // RedirectionInterceptor *interceptor.Redirection - ForwardingClients *cluster.FrontendHTTPClientCache + ClusterMetadata cluster.Metadata + NamespaceRegistry namespace.Registry + Logger log.Logger + MetricsHandler metrics.Handler + Config *Config + CallbackTokenGenerator *commonnexus.CallbackTokenGenerator + HistoryClient resource.HistoryClient + RequestErrorHandler *interceptor.RequestErrorHandler + AuthInterceptor *authorization.Interceptor // required for parsing auth info, not used as an interceptor HTTPTraceProvider commonnexus.HTTPClientTraceProvider - NexusForwarder *nexusForwardingInterceptor - nexusInterceptors []interceptor.NexusInterceptor + nexusInterceptors []interceptornexus.Interceptor clientVersionChecker headers.VersionChecker preProcessErrorsCounter metrics.CounterIface } @@ -73,55 +66,25 @@ func newNexusCompletionHandler( serviceConfig *Config, callbackTokenGenerator *commonnexus.CallbackTokenGenerator, historyClient resource.HistoryClient, - telemetryInterceptor *interceptor.TelemetryInterceptor, requestErrorHandler *interceptor.RequestErrorHandler, - namespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor, - nexusNamespaceRateLimitInterceptor *interceptor.NexusNamespaceRateLimitInterceptor, - namespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor, - rateLimitInterceptor *interceptor.RateLimitInterceptor, authInterceptor *authorization.Interceptor, - redirectionInterceptor *interceptor.Redirection, - forwardingClients *cluster.FrontendHTTPClientCache, httpTraceProvider commonnexus.HTTPClientTraceProvider, - nexusForwarder *nexusForwardingInterceptor, - sdkVersionInterceptor *interceptor.SDKVersionInterceptor, - callerInfoInterceptor *interceptor.CallerInfoInterceptor, - customNexusInterceptors []interceptor.NexusInterceptor, + interceptorsProvider *InterceptorsProvider, + customNexusInterceptors []interceptornexus.Interceptor, ) *nexusCompletionHandler { - nexusInterceptors := []interceptor.NexusInterceptor{ - telemetryInterceptor.InterceptNexus, - authInterceptor.InterceptNexus, - nexusForwarder.InterceptNexus, - namespaceValidationInterceptor.InterceptNexus, - namespaceConcurrencyLimitInterceptor.InterceptNexus, - nexusNamespaceRateLimitInterceptor.InterceptNexus, - rateLimitInterceptor.InterceptNexus, - sdkVersionInterceptor.InterceptNexus, - callerInfoInterceptor.InterceptNexus, - } - // draft-review: check if the customNexusInterceptors should be in the middle of - // the chain instead. Interleaved howver, is out of scope and will not be supported - nexusInterceptors = append(nexusInterceptors, customNexusInterceptors...) + return &nexusCompletionHandler{ - ClusterMetadata: clusterMetadata, - NamespaceRegistry: namespaceRegistry, - Logger: log.With(logger, tag.NexusStageCallerInbound), - MetricsHandler: metricsHandler, - Config: serviceConfig, - CallbackTokenGenerator: callbackTokenGenerator, - HistoryClient: historyClient, - // TelemetryInterceptor: telemetryInterceptor, - RequestErrorHandler: requestErrorHandler, - // NamespaceValidationInterceptor: namespaceValidationInterceptor, - // NamespaceRateLimitInterceptor: namespaceRateLimitInterceptor, - // NamespaceConcurrencyLimitInterceptor: namespaceConcurrencyLimitInterceptor, - // RateLimitInterceptor: rateLimitInterceptor, - AuthInterceptor: authInterceptor, - // RedirectionInterceptor: redirectionInterceptor, - ForwardingClients: forwardingClients, + ClusterMetadata: clusterMetadata, + NamespaceRegistry: namespaceRegistry, + Logger: log.With(logger, tag.NexusStageCallerInbound), + MetricsHandler: metricsHandler, + Config: serviceConfig, + CallbackTokenGenerator: callbackTokenGenerator, + HistoryClient: historyClient, + RequestErrorHandler: requestErrorHandler, + AuthInterceptor: authInterceptor, HTTPTraceProvider: httpTraceProvider, - NexusForwarder: nexusForwarder, - nexusInterceptors: nexusInterceptors, + nexusInterceptors: interceptorsProvider.GetNexusInterceptors(), clientVersionChecker: headers.NewDefaultVersionChecker(), preProcessErrorsCounter: metricsHandler.Counter(metrics.NexusCompletionRequestPreProcessErrors.Name()), } @@ -219,17 +182,21 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus return err } - interceptorInput := interceptor.NewCompleteNexusOpInput(ns.Name().String(), r) - interceptorInput.WithForwardingInfo(interceptor.NexusForwardingInfo{ + interceptorInput := interceptornexus.NewCompleteOpInput(ns.Name().String(), r) + interceptorInput.WithForwardingInfo(interceptornexus.ForwardingInfo{ OriginalRequestHeaders: rCtx.originalHeaders, BusinessID: rCtx.businessID, }) - finalHandler := func(ctx context.Context, _ interceptor.NexusInterceptorInput) (any, error) { + interceptorInput.WithRequestMetadata(interceptornexus.RequestMetadata{ + APIName: nexusCompletionAPIName, + NamespaceEntry: ns, + }) + finalHandler := func(ctx context.Context, _ interceptornexus.InterceptorInput) (any, error) { return nil, h.completeOperationRequest(ctx, logger, completion, r, rCtx) } - _, err = interceptor.ChainNexusInterceptors(finalHandler, h.nexusInterceptors)(ctx, interceptorInput) + _, err = interceptornexus.ChainInterceptors(finalHandler, h.nexusInterceptors)(ctx, interceptorInput) if err != nil { - if taggedErr, ok := errors.AsType[*interceptor.InterceptorError](err); ok { + if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { return taggedErr.Err } return err @@ -464,9 +431,6 @@ type requestContext struct { func (c *requestContext) augmentContext(ctx context.Context, header http.Header) context.Context { ctx = interceptor.WithTelemetryContext(ctx, c) - ctx = interceptor.WithNexusAPIName(ctx, nexusCompletionAPIName) - ctx = interceptor.WithNexusNamespace(ctx, c.namespace) - ctx = interceptor.WithNexusEndpointName(ctx, "") if userAgent := header.Get(headerUserAgent); userAgent != "" { // Preserve original strict behavior: only process if exactly one delimiter present. if strings.Count(userAgent, clientNameVersionDelim) == 1 { diff --git a/service/frontend/nexus_forward_interceptor.go b/service/frontend/nexus_forward_interceptor.go index 3865e576c2d..c9c8936f7fa 100644 --- a/service/frontend/nexus_forward_interceptor.go +++ b/service/frontend/nexus_forward_interceptor.go @@ -19,6 +19,7 @@ import ( commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/rpc/interceptor" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" ) type nexusForwardingInterceptor struct { @@ -54,20 +55,20 @@ func newNexusForwardingInterceptor( func (i *nexusForwardingInterceptor) InterceptNexus( ctx context.Context, - in interceptor.NexusInterceptorInput, - next interceptor.NexusHandlerFunc, + in interceptornexus.InterceptorInput, + next interceptornexus.HandlerFunc, ) (out any, retErr error) { info := in.ForwardingInfo() - header, err := interceptor.NexusHeaderFromInterceptorInput(in) + header, err := interceptornexus.HeaderFromInterceptorInput(in) if err != nil { - return nil, &interceptor.InterceptorError{ + return nil, &interceptornexus.InterceptorError{ Err: err, Outcome: "interceptor_failed", } } - namespaceEntry, err := interceptor.NexusNamespaceFromContext(ctx) + namespaceEntry, err := in.NamespaceEntry() if err != nil { - return nil, &interceptor.InterceptorError{ + return nil, &interceptornexus.InterceptorError{ Err: err, Outcome: "interceptor_failed", } @@ -78,7 +79,7 @@ func (i *nexusForwardingInterceptor) InterceptNexus( return next(ctx, in) } if !i.shouldForwardRequest(ctx, header, namespaceEntry) { - return nil, &interceptor.InterceptorError{ + return nil, &interceptornexus.InterceptorError{ Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive"), Outcome: "namespace_inactive_forwarding_disabled", } @@ -86,31 +87,31 @@ func (i *nexusForwardingInterceptor) InterceptNexus( telemetryContext, err := interceptor.TelemetryContextFromContext(ctx) if err != nil { - return nil, &interceptor.InterceptorError{ + return nil, &interceptornexus.InterceptorError{ Err: err, Outcome: "interceptor_failed", } } telemetryContext.SetMetricsOutcome("request_forwarded") - metricsHandler, forwardStartTime := i.redirectionInterceptor.BeforeCall(interceptor.NexusMethodName(in)) + metricsHandler, forwardStartTime := i.redirectionInterceptor.BeforeCall(interceptornexus.MethodName(in)) defer func() { redirectionErr := retErr - if taggedErr, ok := errors.AsType[*interceptor.InterceptorError](retErr); ok { + if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](retErr); ok { redirectionErr = taggedErr.Err } i.redirectionInterceptor.AfterCall(metricsHandler, forwardStartTime, targetCluster, namespaceEntry.Name().String(), redirectionErr) }() switch request := in.(type) { - case interceptor.StartNexusOpInput: + case interceptornexus.StartOpInput: out, retErr = i.forwardStartOperation(ctx, request, info, namespaceEntry, targetCluster, telemetryContext) - case interceptor.CancelNexusOpInput: + case interceptornexus.CancelOpInput: retErr = i.forwardCancelOperation(ctx, request, info, namespaceEntry, targetCluster, telemetryContext) - case interceptor.CompleteNexusOpInput: + case interceptornexus.CompleteOpInput: retErr = i.forwardCompleteOperation(ctx, request, info, namespaceEntry, targetCluster, telemetryContext) default: - return nil, &interceptor.InterceptorError{ + return nil, &interceptornexus.InterceptorError{ Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "forwarding failed, unknown operation type"), } } @@ -134,8 +135,8 @@ func (i *nexusForwardingInterceptor) shouldForwardRequest( func (i *nexusForwardingInterceptor) forwardStartOperation( ctx context.Context, - request interceptor.StartNexusOpInput, - info interceptor.NexusForwardingInfo, + request interceptornexus.StartOpInput, + info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, telemetryContext interceptor.TelemetryContext, @@ -150,7 +151,7 @@ func (i *nexusForwardingInterceptor) forwardStartOperation( response, err := client.StartOperation(ctx, request.OperationName(), request.StartOperationInput.Reader, request.StartOperationOptions) if err != nil { i.logger.Error("received error from remote cluster for forwarded Nexus start operation request", tag.Error(err)) - return nil, &interceptor.InterceptorError{Err: err, Outcome: "forwarded_request_error"} + return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error"} } if response.Successful != nil { return &nexus.HandlerStartOperationResultSync[any]{Value: response.Successful.Reader}, nil @@ -160,8 +161,8 @@ func (i *nexusForwardingInterceptor) forwardStartOperation( func (i *nexusForwardingInterceptor) forwardCancelOperation( ctx context.Context, - request interceptor.CancelNexusOpInput, - info interceptor.NexusForwardingInfo, + request interceptornexus.CancelOpInput, + info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, telemetryContext interceptor.TelemetryContext, @@ -180,15 +181,15 @@ func (i *nexusForwardingInterceptor) forwardCancelOperation( ctx = i.withForwardingTrace(ctx, "CancelNexusOperation", request.OperationName(), "", info, namespaceEntry, targetCluster) if err := handle.Cancel(ctx, request.CancelOperationOptions); err != nil { i.logger.Error("received error from remote cluster for forwarded Nexus cancel operation request", tag.Error(err)) - return &interceptor.InterceptorError{Err: err, Outcome: "forwarded_request_error"} + return &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error"} } return nil } func (i *nexusForwardingInterceptor) forwardCompleteOperation( ctx context.Context, - request interceptor.CompleteNexusOpInput, - info interceptor.NexusForwardingInfo, + request interceptornexus.CompleteOpInput, + info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, telemetryContext interceptor.TelemetryContext, @@ -196,12 +197,12 @@ func (i *nexusForwardingInterceptor) forwardCompleteOperation( client, err := i.forwardingClients.Get(targetCluster) if err != nil { i.logger.Error("unable to get HTTP client for forward request", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) - return &interceptor.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} + return &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} } forwardURL, err := url.JoinPath(client.BaseURL(), commonnexus.RouteCompletionCallback.Path(namespaceEntry.Name().String())) if err != nil { i.logger.Error("failed to construct forwarding request URL", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.TargetCluster(targetCluster)) - return &interceptor.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} + return &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} } request.CompletionRequest.HTTPRequest.Header.Set(interceptor.DCRedirectionAPIHeaderName, "true") request.CompletionRequest.HTTPRequest.Header.Set(interceptor.DCRedirectionSourceCellHeaderName, i.clusterMetadata.GetCurrentClusterName()) @@ -216,7 +217,7 @@ func (i *nexusForwardingInterceptor) forwardCompleteOperation( HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: client, originalRequestHeaders: info.OriginalRequestHeaders, telemetryContext: telemetryContext}).Do, }).CompleteOperation(ctx, forwardURL, completion) if err != nil { - return &interceptor.InterceptorError{Err: err, Outcome: "forwarded_request_error"} + return &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error"} } return nil } @@ -234,7 +235,7 @@ func completeOperationOptions(request *nexusrpc.CompletionRequest) (nexusrpc.Com func (i *nexusForwardingInterceptor) nexusClientForActiveCluster( service string, - info interceptor.NexusForwardingInfo, + info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, telemetryContext interceptor.TelemetryContext, @@ -242,7 +243,7 @@ func (i *nexusForwardingInterceptor) nexusClientForActiveCluster( httpClient, err := i.forwardingClients.Get(targetCluster) if err != nil { i.logger.Error("failed to forward Nexus request: error creating HTTP client", tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) - return nil, &interceptor.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} + return nil, &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} } var baseURL string if i.serviceConfig.NexusForwardRequestUseEndpoint() && info.EndpointID != "" { @@ -252,7 +253,7 @@ func (i *nexusForwardingInterceptor) nexusClientForActiveCluster( } if err != nil { i.logger.Error("failed to forward Nexus request: error constructing ServiceBaseURL", tag.URL(httpClient.BaseURL()), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.WorkflowTaskQueueName(info.TaskQueue), tag.Error(err)) - return nil, &interceptor.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} + return nil, &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} } return nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: httpClient, originalRequestHeaders: info.OriginalRequestHeaders, telemetryContext: telemetryContext}).Do, @@ -266,7 +267,7 @@ func (i *nexusForwardingInterceptor) withForwardingTrace( method string, operation string, requestID string, - info interceptor.NexusForwardingInfo, + info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, ) context.Context { diff --git a/service/frontend/nexus_forward_interceptor_test.go b/service/frontend/nexus_forward_interceptor_test.go index 4e4df35dc43..01a194c6f20 100644 --- a/service/frontend/nexus_forward_interceptor_test.go +++ b/service/frontend/nexus_forward_interceptor_test.go @@ -23,6 +23,7 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/rpc/interceptor" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" ) func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { @@ -51,13 +52,13 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { Scheme: "http", }, }} - input := interceptor.NewStartNexusOpInput("s", "o", testNamespace, nexus.StartOperationOptions{ + input := interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{ Header: nexus.Header{"X-Request": "request"}, }, nexus.NewLazyValue(nexus.DefaultSerializer(), &nexus.Reader{ ReadCloser: io.NopCloser(bytes.NewBufferString(`"input"`)), Header: nexus.Header{"type": "json"}, })) - input.WithForwardingInfo(interceptor.NexusForwardingInfo{ + input.WithForwardingInfo(interceptornexus.ForwardingInfo{ OriginalRequestHeaders: http.Header{"X-Original": {"original"}}, TaskQueue: "task-queue", }) @@ -138,19 +139,21 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { NexusForwardRequestUseEndpoint: dynamicconfig.GetBoolPropertyFn(false), }, } - ctx := interceptor.WithNexusNamespace(context.Background(), tc.namespace) - ctx = interceptor.WithTelemetryContext(ctx, &forwardingTelemetryContext{}) + input.WithRequestMetadata(interceptornexus.RequestMetadata{ + NamespaceEntry: tc.namespace, + }) + ctx := interceptor.WithTelemetryContext(context.Background(), &forwardingTelemetryContext{}) nextCalled := false result, err := forwarder.InterceptNexus( ctx, input, - func(context.Context, interceptor.NexusInterceptorInput) (any, error) { + func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return requestHandledLocally, nil }, ) if tc.expectedOutcome != "" { - var interceptorErr *interceptor.InterceptorError + var interceptorErr *interceptornexus.InterceptorError require.ErrorAs(t, err, &interceptorErr) require.Equal(t, tc.expectedOutcome, interceptorErr.Outcome) } else { diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index 108220a743d..b14dab872b0 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -31,6 +31,7 @@ import ( commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/rpc/interceptor" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc/metadata" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -75,11 +76,8 @@ type operationContext struct { metricsHandler metrics.Handler logger log.Logger clientVersionChecker headers.VersionChecker - auth *authorization.Interceptor telemetryInterceptor *interceptor.TelemetryInterceptor requestErrorHandler *interceptor.RequestErrorHandler - redirectionInterceptor *interceptor.Redirection - forwardingEnabledForNamespace dynamicconfig.BoolPropertyFnWithNamespaceFilter headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig] } @@ -95,9 +93,6 @@ func (c *operationContext) matchingRequest(req *nexuspb.Request) *matchingservic func (c *operationContext) augmentContext(ctx context.Context, header nexus.Header) context.Context { ctx = interceptor.WithTelemetryContext(ctx, c) - ctx = interceptor.WithNexusAPIName(ctx, c.apiName) - ctx = interceptor.WithNexusEndpointName(ctx, c.endpointName) - ctx = interceptor.WithNexusNamespace(ctx, c.namespace) if userAgent, ok := header[headerUserAgent]; ok { // Use SplitN for efficiency but enforce exactly one delimiter to preserve the // original (pre-SplitN) strictness where additional delimiters cause us to ignore @@ -139,9 +134,6 @@ func (c *operationContext) SetFailureSource(source string) { c.setFailureSource(source) } -// replaces registerRequestErrorHandler and the cleanupFunctions -// registry. Errors that a worker produced are already reported by the worker's own -// cluster, so only other sources are handled here. func (c *operationContext) HandleRequestError(err error) { if err == nil { return @@ -235,23 +227,18 @@ type nexusContextKey struct{} // Dispatches Nexus requests as Nexus tasks to workers via matching. type nexusHandler struct { nexus.UnimplementedHandler - logger log.Logger - metricsHandler metrics.Handler - clusterMetadata cluster.Metadata - namespaceRegistry namespace.Registry - matchingClient matchingservice.MatchingServiceClient - auth *authorization.Interceptor - // telemetryInterceptor *interceptor.TelemetryInterceptor - // requestErrorHandler *interceptor.RequestErrorHandler - // redirectionInterceptor *interceptor.Redirection - forwardingEnabledForNamespace dynamicconfig.BoolPropertyFnWithNamespaceFilter - forwardingClients *cluster.FrontendHTTPClientCache - payloadSizeLimit dynamicconfig.IntPropertyFnWithNamespaceFilter - headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] - useForwardByEndpoint dynamicconfig.BoolPropertyFn - metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig] - httpTraceProvider commonnexus.HTTPClientTraceProvider - nexusInterceptors []interceptor.NexusInterceptor + logger log.Logger + metricsHandler metrics.Handler + clusterMetadata cluster.Metadata + namespaceRegistry namespace.Registry + matchingClient matchingservice.MatchingServiceClient + requestErrorHandler *interceptor.RequestErrorHandler + payloadSizeLimit dynamicconfig.IntPropertyFnWithNamespaceFilter + headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] + useForwardByEndpoint dynamicconfig.BoolPropertyFn + metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig] + httpTraceProvider commonnexus.HTTPClientTraceProvider + nexusInterceptors []interceptornexus.Interceptor } // Extracts a nexusContext from the given ctx and returns an operationContext with tagged metrics and logging. @@ -266,13 +253,9 @@ func (h *nexusHandler) getOperationContext(ctx context.Context, method string) ( method: method, clusterMetadata: h.clusterMetadata, clientVersionChecker: headers.NewDefaultVersionChecker(), - auth: h.auth, - // telemetryInterceptor: h.telemetryInterceptor, - // requestErrorHandler: h.requestErrorHandler, - // redirectionInterceptor: h.redirectionInterceptor, - forwardingEnabledForNamespace: h.forwardingEnabledForNamespace, - headersBlacklist: h.headersBlacklist, - metricTagConfig: h.metricTagConfig, + requestErrorHandler: h.requestErrorHandler, + headersBlacklist: h.headersBlacklist, + metricTagConfig: h.metricTagConfig, } oc.metricsHandlerForInterceptors = h.metricsHandler.WithTags( metrics.OperationTag(method), @@ -298,7 +281,6 @@ func (h *nexusHandler) getOperationContext(ctx context.Context, method string) ( } return nil, commonnexus.ConvertGRPCError(err, false) } - oc.forwardingEnabledForNamespace = h.forwardingEnabledForNamespace oc.logger = log.With(h.logger, tag.Operation(method), tag.WorkflowNamespace(nc.namespaceName)) return &oc, nil } @@ -346,21 +328,26 @@ func (h *nexusHandler) StartOperation( }, }) - finalHandler := func(ctx context.Context, _ interceptor.NexusInterceptorInput) (any, error) { + finalHandler := func(ctx context.Context, _ interceptornexus.InterceptorInput) (any, error) { return h.finalStartHandler(ctx, oc, operation, input, &startOperationRequest, request) } - nexusOpInput := interceptor.NewStartNexusOpInput(service, operation, oc.namespaceName, options, input) - nexusOpInput.WithForwardingInfo(interceptor.NexusForwardingInfo{ + nexusOpInput := interceptornexus.NewStartOpInput(service, operation, oc.namespaceName, options, input) + nexusOpInput.WithForwardingInfo(interceptornexus.ForwardingInfo{ OriginalRequestHeaders: oc.originalRequestHeaders, TaskQueue: oc.taskQueue, EndpointID: oc.endpointID, EndpointName: oc.endpointName, }) - chainedHandler := interceptor.ChainNexusInterceptors(finalHandler, h.nexusInterceptors) + nexusOpInput.WithRequestMetadata(interceptornexus.RequestMetadata{ + APIName: oc.apiName, + NamespaceEntry: oc.namespace, + EndpointName: oc.endpointName, + }) + chainedHandler := interceptornexus.ChainInterceptors(finalHandler, h.nexusInterceptors) out, err := chainedHandler(ctx, nexusOpInput) if err != nil { - if taggedErr, ok := errors.AsType[*interceptor.InterceptorError](err); ok { + if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { return nil, taggedErr.Err } return nil, err @@ -455,21 +442,26 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, }, }) - finalHandler := func(ctx context.Context, _ interceptor.NexusInterceptorInput) (any, error) { + finalHandler := func(ctx context.Context, _ interceptornexus.InterceptorInput) (any, error) { return nil, h.finalCancelHandler(ctx, oc, operation, request) } - nexusInterceptorInput := interceptor.NewCancelNexusOpInput(service, operation, oc.namespaceName, options, token) - nexusInterceptorInput.WithForwardingInfo(interceptor.NexusForwardingInfo{ + nexusInterceptorInput := interceptornexus.NewCancelOpInput(service, operation, oc.namespaceName, options, token) + nexusInterceptorInput.WithForwardingInfo(interceptornexus.ForwardingInfo{ OriginalRequestHeaders: oc.originalRequestHeaders, TaskQueue: oc.taskQueue, EndpointID: oc.endpointID, EndpointName: oc.endpointName, }) - chainedHandler := interceptor.ChainNexusInterceptors(finalHandler, h.nexusInterceptors) + nexusInterceptorInput.WithRequestMetadata(interceptornexus.RequestMetadata{ + APIName: oc.apiName, + NamespaceEntry: oc.namespace, + EndpointName: oc.endpointName, + }) + chainedHandler := interceptornexus.ChainInterceptors(finalHandler, h.nexusInterceptors) _, err = chainedHandler(ctx, nexusInterceptorInput) if err != nil { - if taggedErr, ok := errors.AsType[*interceptor.InterceptorError](err); ok { + if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { return taggedErr.Err } return err diff --git a/service/frontend/nexus_handler_test.go b/service/frontend/nexus_handler_test.go index 7f2cbdfbf28..b553222e3bf 100644 --- a/service/frontend/nexus_handler_test.go +++ b/service/frontend/nexus_handler_test.go @@ -9,10 +9,8 @@ import ( enumspb "go.temporal.io/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/authorization" - "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/cluster/clustertest" - "go.temporal.io/server/common/config" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" @@ -110,21 +108,6 @@ func newOperationContext(options contextOptions) *operationContext { 1, ) - checker := mockNamespaceChecker(oc.namespace.Name()) - oc.auth = authorization.NewInterceptor( - nil, - mockAuthorizer{}, - oc.metricsHandler, - oc.logger, - checker, - nil, - "", - "", - dynamicconfig.GetBoolPropertyFn(false), // exposeAuthorizerErrors - dynamicconfig.GetBoolPropertyFn(false), // enableCrossNamespaceCommands - dynamicconfig.GetBoolPropertyFnFilteredByNamespace(false), // enablePrincipalPropagation - dynamicconfig.GetBoolPropertyFn(false), // disableStreamingAuthorizer - ) oc.namespaceConcurrencyLimitInterceptor = interceptor.NewConcurrentRequestLimitInterceptor( nil, nil, @@ -150,25 +133,11 @@ func newOperationContext(options contextOptions) *operationContext { oc.clusterMetadata = clustertest.NewMetadataForTest( cluster.NewTestClusterMetadataConfig(true, !options.namespacePassive), ) - oc.forwardingEnabledForNamespace = dynamicconfig.GetBoolPropertyFnFilteredByNamespace( - options.redirectAllow, - ) re, err := dynamicconfig.ConvertWildcardStringListToRegexp(options.headersBlacklist) if err != nil { panic(err) // nolint:forbidigo } oc.headersBlacklist = dynamicconfig.GetTypedPropertyFn(re) - oc.redirectionInterceptor = interceptor.NewRedirection( - nil, - dynamicconfig.GetBoolPropertyFnFilteredByNamespace(false), - nil, - config.DCRedirectionPolicy{Policy: interceptor.DCRedirectionPolicyAllAPIsForwarding}, - oc.logger, - nil, - oc.metricsHandlerForInterceptors, - clock.NewRealTimeSource(), - oc.clusterMetadata, - ) return oc } diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index 7dc5a85868e..cecd8cdb686 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -27,6 +27,7 @@ import ( "go.temporal.io/server/common/routing" "go.temporal.io/server/common/rpc" "go.temporal.io/server/common/rpc/interceptor" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/telemetry" "go.temporal.io/server/service/frontend/configs" "google.golang.org/grpc/codes" @@ -64,37 +65,20 @@ func NewNexusOperationHTTPHandler( redirectionInterceptor *interceptor.Redirection, namespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor, namespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor, - nexusNamespaceRateLimitInterceptor *interceptor.NexusNamespaceRateLimitInterceptor, + nexusNamespaceRateLimitInterceptor *interceptor.NamespaceRateLimitInterceptorWrapper, namespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor, rateLimitInterceptor *interceptor.RateLimitInterceptor, sdkVersionInterceptor *interceptor.SDKVersionInterceptor, callerInfoInterceptor *interceptor.CallerInfoInterceptor, nexusForwarder *nexusForwardingInterceptor, - customNexusInterceptors []interceptor.NexusInterceptor, + interceptorsProvider *InterceptorsProvider, + customNexusInterceptors []interceptornexus.Interceptor, logger log.Logger, httpTraceProvider commonnexus.HTTPClientTraceProvider, httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter, ) *NexusOperationHTTPHandler { logger = log.With(logger, tag.NexusStageHandlerInbound) - // draft-review: should we also just make an interceptors provider fx - // so it can be shared/declared in a single place. Maybe not worth it as - // eventual goal is to remove the completion handler and move that into the - // http handler as well - - nexusInterceptors := []interceptor.NexusInterceptor{ - telemetryInterceptor.InterceptNexus, - authInterceptor.InterceptNexus, - nexusForwarder.InterceptNexus, - namespaceValidationInterceptor.InterceptNexus, - namespaceConcurrencyLimitInterceptor.InterceptNexus, - nexusNamespaceRateLimitInterceptor.InterceptNexus, - rateLimitInterceptor.InterceptNexus, - sdkVersionInterceptor.InterceptNexus, - callerInfoInterceptor.InterceptNexus, - } - nexusInterceptors = append(nexusInterceptors, customNexusInterceptors...) - return &NexusOperationHTTPHandler{ base: nexusrpc.BaseHTTPHandler{ Logger: log.NewSlogLogger(logger), @@ -112,23 +96,18 @@ func NewNexusOperationHTTPHandler( httpServerHandlerInstrumenter: httpServerHandlerInstrumenter, nexusHandler: nexusrpc.NewHTTPHandler(nexusrpc.HandlerOptions{ Handler: &nexusHandler{ - logger: logger, - metricsHandler: metricsHandler, - clusterMetadata: clusterMetadata, - namespaceRegistry: namespaceRegistry, - matchingClient: matchingservice.MatchingServiceClient(matchingClient), - auth: authInterceptor, - // telemetryInterceptor: telemetryInterceptor, - // requestErrorHandler: requestErrorHandler, - // redirectionInterceptor: redirectionInterceptor, - forwardingEnabledForNamespace: serviceConfig.EnableNamespaceNotActiveAutoForwarding, - forwardingClients: clientCache, - payloadSizeLimit: serviceConfig.BlobSizeLimitError, - headersBlacklist: serviceConfig.NexusRequestHeadersBlacklist, - useForwardByEndpoint: serviceConfig.NexusForwardRequestUseEndpoint, - metricTagConfig: serviceConfig.NexusOperationsMetricTagConfig, - httpTraceProvider: httpTraceProvider, - nexusInterceptors: nexusInterceptors, + logger: logger, + metricsHandler: metricsHandler, + clusterMetadata: clusterMetadata, + namespaceRegistry: namespaceRegistry, + matchingClient: matchingservice.MatchingServiceClient(matchingClient), + requestErrorHandler: requestErrorHandler, + payloadSizeLimit: serviceConfig.BlobSizeLimitError, + headersBlacklist: serviceConfig.NexusRequestHeadersBlacklist, + useForwardByEndpoint: serviceConfig.NexusForwardRequestUseEndpoint, + metricTagConfig: serviceConfig.NexusOperationsMetricTagConfig, + httpTraceProvider: httpTraceProvider, + nexusInterceptors: interceptorsProvider.GetNexusInterceptors(), }, GetResultTimeout: serviceConfig.KeepAliveMaxConnectionIdle(), Logger: log.NewSlogLogger(logger), diff --git a/service/fx.go b/service/fx.go index f41031b62a4..30490aa1b3d 100644 --- a/service/fx.go +++ b/service/fx.go @@ -165,7 +165,7 @@ func getUnaryInterceptors(params GrpcServerOptionsParams) []grpc.UnaryServerInte params.ServiceErrorInterceptor.Intercept, metrics.NewServerMetricsContextInjectorInterceptor(), metrics.NewServerMetricsTrailerPropagatorInterceptor(params.Logger), - params.TelemetryInterceptor.UnaryIntercept, + params.TelemetryInterceptor.Intercept, } interceptors = append(interceptors, params.AdditionalInterceptors...) diff --git a/service/history/history_engine_test.go b/service/history/history_engine_test.go index 7550fdcfad9..05a2751ad13 100644 --- a/service/history/history_engine_test.go +++ b/service/history/history_engine_test.go @@ -5588,7 +5588,7 @@ func (s *engineSuite) TestEagerWorkflowStart_DoesNotCreateTransferTask() { s.mockShard.Resource.Logger, s.config.LogAllReqErrors, s.mockErrorHandler) - response, err := i.UnaryIntercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req any) (any, error) { + response, err := i.Intercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req any) (any, error) { response, err := s.historyEngine.StartWorkflowExecution(ctx, &historyservice.StartWorkflowExecutionRequest{ NamespaceId: tests.NamespaceID.String(), Attempt: 1, @@ -5627,7 +5627,7 @@ func (s *engineSuite) TestEagerWorkflowStart_FromCron_SkipsEager() { s.mockShard.Resource.Logger, s.config.LogAllReqErrors, s.mockErrorHandler) - response, err := i.UnaryIntercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req any) (any, error) { + response, err := i.Intercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req any) (any, error) { firstWorkflowTaskBackoff := time.Second response, err := s.historyEngine.StartWorkflowExecution(ctx, &historyservice.StartWorkflowExecutionRequest{ NamespaceId: tests.NamespaceID.String(), @@ -5671,7 +5671,7 @@ func (s *engineSuite) TestEagerWorkflowStart_WithSearchAttributes() { s.mockShard.Resource.Logger, s.config.LogAllReqErrors, s.mockErrorHandler) - response, err := i.UnaryIntercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req any) (any, error) { + response, err := i.Intercept(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "StartWorkflowExecution"}, func(ctx context.Context, req any) (any, error) { response, err := s.historyEngine.StartWorkflowExecution(ctx, &historyservice.StartWorkflowExecutionRequest{ NamespaceId: tests.NamespaceID.String(), Attempt: 1, diff --git a/temporal/fx.go b/temporal/fx.go index 9471132aa0d..dd70640dc23 100644 --- a/temporal/fx.go +++ b/temporal/fx.go @@ -47,7 +47,7 @@ import ( "go.temporal.io/server/common/resource" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" - rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" "go.temporal.io/server/common/telemetry" @@ -122,7 +122,7 @@ type ( TokenProvider auth.TokenProvider ServiceHosts map[primitives.ServiceName]static.Hosts - CustomFrontendNexusInterceptors []rpcinterceptor.NexusInterceptor + CustomFrontendNexusInterceptors []nexus.Interceptor // below are things that could be over write by server options or may have default if not supplied by serverOptions. Logger log.Logger @@ -326,7 +326,7 @@ func ServerOptionsProvider(opts []ServerOption) (serverOptionsProvider, error) { CustomVisibilityStore: so.customVisibilityStoreFactory, CustomHistoryArchiverFactory: so.customHistoryArchiverFactory, CustomVisibilityArchiverFactory: so.customVisibilityArchiverFactory, - CustomFrontendNexusInterceptors: so.customFrontendNexusInterceptors, + CustomFrontendNexusInterceptors: so.customFrontendUnifiedInterceptors, SearchAttributesMapper: so.searchAttributesMapper, CustomFrontendInterceptors: so.customFrontendInterceptors, @@ -401,7 +401,7 @@ type ( PersistenceFactoryProvider persistenceClient.FactoryProviderFn SearchAttributesMapper searchattribute.Mapper CustomFrontendInterceptors []grpc.UnaryServerInterceptor - CustomFrontendNexusInterceptors []rpcinterceptor.NexusInterceptor + CustomFrontendNexusInterceptors []nexus.Interceptor AdditionalStreamInterceptors []grpc.StreamServerInterceptor Authorizer authorization.Authorizer ClaimMapper authorization.ClaimMapper diff --git a/temporal/server_option.go b/temporal/server_option.go index fd1cbd41782..a86e5daa151 100644 --- a/temporal/server_option.go +++ b/temporal/server_option.go @@ -18,7 +18,7 @@ import ( "go.temporal.io/server/common/resolver" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" - rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/testing/testhooks" "google.golang.org/grpc" @@ -210,14 +210,14 @@ func WithChainedFrontendGrpcInterceptors( }) } -// WithChainedFrontendNexusInterceptors sets an orderered chain of custom Nexus interceptors that will be invoked for -// Frontend Nexus API calls. The custom interceptors will be appended to the end of the internal ServerInterceptors -// and invoked in the order that they appear in the supplied list. +// TBD: this will become unified interceptors instead +// +//nolint:staticcheck func WithChainedFrontendNexusInterceptors( - interceptors ...rpcinterceptor.NexusInterceptor, + interceptors ...nexus.Interceptor, ) ServerOption { return applyFunc(func(s *serverOptions) { - s.customFrontendNexusInterceptors = interceptors + s.customFrontendUnifiedInterceptors = interceptors }) } diff --git a/temporal/server_options.go b/temporal/server_options.go index ff1086e7a94..b161d241e91 100644 --- a/temporal/server_options.go +++ b/temporal/server_options.go @@ -21,7 +21,7 @@ import ( "go.temporal.io/server/common/resolver" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" - rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" + "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/testing/testhooks" "google.golang.org/grpc" @@ -45,29 +45,29 @@ type ( startupSynchronizationMode synchronizationModeParams - logger log.Logger - namespaceLogger log.Logger - authorizer authorization.Authorizer - tlsConfigProvider encryption.TLSConfigProvider - claimMapper authorization.ClaimMapper - audienceGetter authorization.JWTAudienceMapper - persistenceServiceResolver resolver.ServiceResolver - elasticsearchHttpClient *http.Client //nolint:staticcheck // should be elasticsearchHTTPClient - dynamicConfigClient dynamicconfig.Client - customDataStoreFactory persistenceClient.AbstractDataStoreFactory - customVisibilityStoreFactory visibility.VisibilityStoreFactory - customHistoryArchiverFactory provider.CustomHistoryArchiverFactory - customVisibilityArchiverFactory provider.CustomVisibilityArchiverFactory - clientFactoryProvider client.FactoryProvider - persistenceFactoryProvider persistenceClient.FactoryProviderFn - searchAttributesMapper searchattribute.Mapper - customFrontendInterceptors []grpc.UnaryServerInterceptor - customFrontendNexusInterceptors []rpcinterceptor.NexusInterceptor - additionalStreamInterceptors []grpc.StreamServerInterceptor - metricHandler metrics.Handler - eventLoggerProvider otellog.LoggerProvider - tokenProvider auth.TokenProvider - testHooks *testhooks.TestHooks + logger log.Logger + namespaceLogger log.Logger + authorizer authorization.Authorizer + tlsConfigProvider encryption.TLSConfigProvider + claimMapper authorization.ClaimMapper + audienceGetter authorization.JWTAudienceMapper + persistenceServiceResolver resolver.ServiceResolver + elasticsearchHttpClient *http.Client //nolint:staticcheck // should be elasticsearchHTTPClient + dynamicConfigClient dynamicconfig.Client + customDataStoreFactory persistenceClient.AbstractDataStoreFactory + customVisibilityStoreFactory visibility.VisibilityStoreFactory + customHistoryArchiverFactory provider.CustomHistoryArchiverFactory + customVisibilityArchiverFactory provider.CustomVisibilityArchiverFactory + clientFactoryProvider client.FactoryProvider + persistenceFactoryProvider persistenceClient.FactoryProviderFn + searchAttributesMapper searchattribute.Mapper + customFrontendInterceptors []grpc.UnaryServerInterceptor + customFrontendUnifiedInterceptors []nexus.Interceptor + additionalStreamInterceptors []grpc.StreamServerInterceptor + metricHandler metrics.Handler + eventLoggerProvider otellog.LoggerProvider + tokenProvider auth.TokenProvider + testHooks *testhooks.TestHooks } ) diff --git a/tools/flakereport/report.go b/tools/flakereport/report.go index 874e9ca57a6..18a3266d9ed 100644 --- a/tools/flakereport/report.go +++ b/tools/flakereport/report.go @@ -148,7 +148,7 @@ func generateOccurrenceReportTable(reports []TestReport, nameHeader, countHeader reports = limitReportRows(reports) var sb strings.Builder - sb.WriteString(fmt.Sprintf("| %s | %s | Last Occurrence | Trend | Links |\n", nameHeader, countHeader)) + fmt.Fprintf(&sb, "| %s | %s | Last Occurrence | Trend | Links |\n", nameHeader, countHeader) sb.WriteString("|------|--------------------|-----------------|-------|-------|\n") for _, report := range reports { links := formatLinks(report.GitHubURLs, maxLinks) @@ -156,8 +156,8 @@ func generateOccurrenceReportTable(reports []TestReport, nameHeader, countHeader if !report.LastFailure.IsZero() { lastOccurrence = hoursAgo(report.LastFailure) } - sb.WriteString(fmt.Sprintf("| `%s` | %d | %s | `%s` | %s |\n", - report.TestName, report.FailureCount, lastOccurrence, formatSparkline(report.TrendPoints), links)) + fmt.Fprintf(&sb, "| `%s` | %d | %s | `%s` | %s |\n", + report.TestName, report.FailureCount, lastOccurrence, formatSparkline(report.TrendPoints), links) } return sb.String() From a127acb2c27a6f6e7e8e89cc310f328542f9b843 Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Thu, 20 Aug 2026 15:13:56 -0700 Subject: [PATCH 09/12] WIP: amending nexus interceptor tests to fall in-line with gRPC interceptor chain --- common/rpc/interceptor/telemetry.go | 5 ++- service/frontend/frontend_interceptors.go | 35 ++------------------ tests/nexus_api_validation_test.go | 10 +++--- tests/nexus_workflow_test.go | 16 ++++----- tests/xdc/nexus_request_forwarding_test.go | 38 +++++++++++----------- 5 files changed, 36 insertions(+), 68 deletions(-) diff --git a/common/rpc/interceptor/telemetry.go b/common/rpc/interceptor/telemetry.go index ddbd1fd310c..ebb918427e0 100644 --- a/common/rpc/interceptor/telemetry.go +++ b/common/rpc/interceptor/telemetry.go @@ -238,9 +238,8 @@ func TelemetryContextFromContext(ctx context.Context) (TelemetryContext, error) } // InterceptNexus records request metrics and recovers panics for a Nexus request. -// It runs outermost in the chain, so metrics are recorded in the source cluster even -// when the forwarder redirects. Forwarded requests are distinguished by the -// "request_forwarded" outcome tag rather than by being omitted. +// It runs after auth and redirection, mirroring the gRPC chain, so requests rejected or +// forwarded by those interceptors are not counted here // It also publishes the metrics context that downstream interceptors read via // GetMetricsHandlerFromContext. func (ti *TelemetryInterceptor) InterceptNexus( diff --git a/service/frontend/frontend_interceptors.go b/service/frontend/frontend_interceptors.go index 01aab2ec2a9..48597628ac6 100644 --- a/service/frontend/frontend_interceptors.go +++ b/service/frontend/frontend_interceptors.go @@ -32,10 +32,7 @@ type Interceptor interface { } type InterceptorsProvider struct { - interceptors []Interceptor - // nexusInterceptors are different from interceptors due to a few reasons - // eventual goal is to have exact same set and order - ninterceptors []Interceptor + interceptors []Interceptor retryableInterceptor *interceptor.RetryableInterceptor // required to be last in chain after custom interceptors customGRPCInterceptors []grpc.UnaryServerInterceptor // required for legacy reasons faultGenerator grpcfaults.Generator @@ -77,37 +74,10 @@ func NewInterceptorsProvider( namespaceStateValidatorInterceptor, namespaceLogInterceptor, metricsCtxInjectorInterceptor, - // for gRPC, auth is before telemetry authInterceptor, namespaceHandoverInterceptor, redirectionSlot, telemetryInterceptor, - // rest of the chain is identical for gRPC and Nexus - healthInterceptor, - namespaceValidatorInterceptor, - namespaceCountLimiterInterceptor, - namespaceRateLimiterInterceptorWrapper, - rateLimitInterceptor, - sdkVersionInterceptor, - callerInfoInterceptor, - slowRequestLoggerInterceptor, - chasmRequestVisibilityInterceptor, - contextMetadataInterceptor, - } - ninterceptors := []Interceptor{ - maskInternalErrorDetailsInterceptor, - serviceErrorInterceptor, - frontendServiceErrorInterceptor, - businessIDInterceptor, - namespaceStateValidatorInterceptor, - namespaceLogInterceptor, - metricsCtxInjectorInterceptor, - // for Nexus, telemetry is before auth - telemetryInterceptor, - authInterceptor, - namespaceHandoverInterceptor, - redirectionSlot, - // rest of the chain is identical for gRPC and Nexus healthInterceptor, namespaceValidatorInterceptor, namespaceCountLimiterInterceptor, @@ -126,7 +96,6 @@ func NewInterceptorsProvider( return &InterceptorsProvider{ interceptors: interceptors, - ninterceptors: ninterceptors, customGRPCInterceptors: customGRPCInterceptors, retryableInterceptor: retryableInterceptor, faultGenerator: grpcfaultstest.NewGenerator(testHooks), @@ -150,7 +119,7 @@ func (n *InterceptorsProvider) GetInterceptors() []grpc.UnaryServerInterceptor { func (n *InterceptorsProvider) GetNexusInterceptors() []nexus.Interceptor { nexusInterceptors := []nexus.Interceptor{} - for _, i := range n.ninterceptors { + for _, i := range n.interceptors { nexusInterceptors = append(nexusInterceptors, i.InterceptNexus) } diff --git a/tests/nexus_api_validation_test.go b/tests/nexus_api_validation_test.go index dde819faddf..354145a2994 100644 --- a/tests/nexus_api_validation_test.go +++ b/tests/nexus_api_validation_test.go @@ -201,7 +201,7 @@ func (s *NexusAPIValidationTestSuite) TestNexusStartOperation_Forbidden() { client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{BaseURL: dispatchURL, Service: "test-service"}) s.NoError(err) - capture := env.StartNamespaceMetricCapture() + // capture := env.StartNamespaceMetricCapture() _, err = nexusrpc.StartOperation(s.Context(), client, op, "input", nexus.StartOperationOptions{}) @@ -209,10 +209,10 @@ func (s *NexusAPIValidationTestSuite) TestNexusStartOperation_Forbidden() { s.ErrorAs(err, &handlerErr) tc.checkFailure(s, handlerErr) - requests := capture.Metric("nexus_requests") - s.Len(requests, 1) - s.Subset(requests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "method": "StartNexusOperation", "outcome": tc.expectedOutcomeMetric}) - s.Equal(int64(1), requests[0].Value) + // requests := capture.Metric("nexus_requests") + // s.Len(requests, 1) + // s.Subset(requests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "method": "StartNexusOperation", "outcome": tc.expectedOutcomeMetric}) + // s.Equal(int64(1), requests[0].Value) } for _, tc := range testCases { diff --git a/tests/nexus_workflow_test.go b/tests/nexus_workflow_test.go index abef91f61e1..8adb7985cdb 100644 --- a/tests/nexus_workflow_test.go +++ b/tests/nexus_workflow_test.go @@ -1782,14 +1782,14 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationAsyncCompletionAuthErrors(cha } publicCallbackURL := "http://" + env.HttpAPIAddress() + "/" + commonnexus.RouteCompletionCallback.Path(env.Namespace().String()) - capture := env.StartNamespaceMetricCapture() + // capture := env.StartNamespaceMetricCapture() err = s.sendNexusCompletionRequest(s.Context(), publicCallbackURL, completion) - completionRequests := capture.Metric("nexus_completion_requests") + // completionRequests := capture.Metric("nexus_completion_requests") var handlerErr *nexus.HandlerError s.ErrorAs(err, &handlerErr) s.Equal(nexus.HandlerErrorTypeUnauthorized, handlerErr.Type) - s.Len(completionRequests, 1) - s.Subset(completionRequests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "outcome": "unauthorized"}) + // s.Len(completionRequests, 1) + // s.Subset(completionRequests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "outcome": "unauthorized"}) } func (s *NexusWorkflowTestSuite) TestNexusOperationAsyncCompletionAuthErrorsNoIdentifier(chasmEnabled bool) { @@ -1812,14 +1812,14 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationAsyncCompletionAuthErrorsNoId Header: nexus.Header{commonnexus.CallbackTokenHeader: callbackToken}, } publicCallbackURL := "http://" + env.HttpAPIAddress() + commonnexus.PathCompletionCallbackNoIdentifier - capture := env.StartNamespaceMetricCapture() + // capture := env.StartNamespaceMetricCapture() err = s.sendNexusCompletionRequest(s.Context(), publicCallbackURL, completion) - completionRequests := capture.Metric("nexus_completion_requests") + // completionRequests := capture.Metric("nexus_completion_requests") var handlerErr *nexus.HandlerError s.ErrorAs(err, &handlerErr) s.Equal(nexus.HandlerErrorTypeUnauthorized, handlerErr.Type) - s.Len(completionRequests, 1) - s.Subset(completionRequests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "outcome": "unauthorized"}) + // s.Len(completionRequests, 1) + // s.Subset(completionRequests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "outcome": "unauthorized"}) } func (s *NexusWorkflowTestSuite) TestNexusOperationAsyncCompletionInternalAuth(chasmEnabled bool) { diff --git a/tests/xdc/nexus_request_forwarding_test.go b/tests/xdc/nexus_request_forwarding_test.go index d3544598782..359fe42bc78 100644 --- a/tests/xdc/nexus_request_forwarding_test.go +++ b/tests/xdc/nexus_request_forwarding_test.go @@ -131,7 +131,7 @@ func (s *NexusRequestForwardingSuite) TestStartOperationForwardedFromStandbyToAc require.NoError(t, retErr) require.Equal(t, "input", result.Successful) requireExpectedMetricsCaptured(t, activeSnap, ns, "StartNexusOperation", "sync_success") - requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "request_forwarded") + // requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "request_forwarded") }, }, { @@ -180,7 +180,7 @@ func (s *NexusRequestForwardingSuite) TestStartOperationForwardedFromStandbyToAc require.NoError(t, json.Unmarshal(appErrDetails.Details, &details)) require.Equal(t, "details", details) requireExpectedMetricsCaptured(t, activeSnap, ns, "StartNexusOperation", "operation_error") - requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "forwarded_request_error") + // requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "forwarded_request_error") }, }, { @@ -203,7 +203,7 @@ func (s *NexusRequestForwardingSuite) TestStartOperationForwardedFromStandbyToAc require.Error(t, handlerErr.Cause) require.Equal(t, "deliberate internal failure", handlerErr.Cause.Error()) requireExpectedMetricsCaptured(t, activeSnap, ns, "StartNexusOperation", "handler_error:INTERNAL") - requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "forwarded_request_error") + // requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "forwarded_request_error") }, }, { @@ -222,7 +222,7 @@ func (s *NexusRequestForwardingSuite) TestStartOperationForwardedFromStandbyToAc require.ErrorAs(t, retErr, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeUnavailable, handlerErr.Type) require.Equal(t, "cluster inactive", handlerErr.Message) - requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "namespace_inactive_forwarding_disabled") + // requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "namespace_inactive_forwarding_disabled") }, }, } @@ -305,7 +305,7 @@ func (s *NexusRequestForwardingSuite) TestCancelOperationForwardedFromStandbyToA assertion: func(t *testing.T, retErr error, activeSnap map[string][]*metricstest.CapturedRecording, passiveSnap map[string][]*metricstest.CapturedRecording) { require.NoError(t, retErr) requireExpectedMetricsCaptured(t, activeSnap, ns, "CancelNexusOperation", "success") - requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "request_forwarded") + // requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "request_forwarded") }, }, { @@ -328,7 +328,7 @@ func (s *NexusRequestForwardingSuite) TestCancelOperationForwardedFromStandbyToA require.Error(t, handlerErr.Cause) require.Equal(t, "deliberate internal failure", handlerErr.Cause.Error()) requireExpectedMetricsCaptured(t, activeSnap, ns, "CancelNexusOperation", "handler_error:INTERNAL") - requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "forwarded_request_error") + // requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "forwarded_request_error") }, }, { @@ -347,7 +347,7 @@ func (s *NexusRequestForwardingSuite) TestCancelOperationForwardedFromStandbyToA require.ErrorAs(t, retErr, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeUnavailable, handlerErr.Type) require.Equal(t, "cluster inactive", handlerErr.Message) - requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "namespace_inactive_forwarding_disabled") + // requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "namespace_inactive_forwarding_disabled") }, }, } @@ -607,18 +607,18 @@ func (s *NexusRequestForwardingSuite) TestOperationCompletionForwardedFromStandb completion.Header.Set(cnexus.CallbackTokenHeader, callbackToken) snap, err := s.sendNexusCompletionRequest(ctx, s.T(), s.clusters[1], publicCallbackUrl, completion) s.NoError(err) - s.Len(snap["nexus_completion_requests"], 1) - s.Subset(snap["nexus_completion_requests"][0].Tags, map[string]string{"namespace": ns, "outcome": "request_forwarded"}) - - // Ensure that CompleteOperation request is tracked as part of normal service telemetry metrics - s.Condition(func() bool { - for _, m := range snap["service_requests"] { - if opTag, ok := m.Tags["operation"]; ok && opTag == "CompleteNexusOperation" { - return true - } - } - return false - }) + // s.Len(snap["nexus_completion_requests"], 1) + // s.Subset(snap["nexus_completion_requests"][0].Tags, map[string]string{"namespace": ns, "outcome": "request_forwarded"}) + + // // Ensure that CompleteOperation request is tracked as part of normal service telemetry metrics + // s.Condition(func() bool { + // for _, m := range snap["service_requests"] { + // if opTag, ok := m.Tags["operation"]; ok && opTag == "CompleteNexusOperation" { + // return true + // } + // } + // return false + // }) // Resend the request and verify we get a not found error since the operation has already completed. snap, err = s.sendNexusCompletionRequest(ctx, s.T(), s.clusters[0], publicCallbackUrl, completion) From cc5607d595c86e582545be6f733a9c82ffc31941 Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Mon, 24 Aug 2026 22:23:47 -0700 Subject: [PATCH 10/12] address feedback --- common/authorization/interceptor.go | 2 +- common/authorization/interceptor_test.go | 11 +- common/rpc/interceptor/caller_info.go | 2 +- common/rpc/interceptor/caller_info_test.go | 16 +- .../interceptor/concurrent_request_limit.go | 3 +- .../concurrent_request_limit_test.go | 26 +- .../context_metadata_interceptor.go | 10 +- common/rpc/interceptor/health.go | 4 +- common/rpc/interceptor/namespace_handover.go | 4 - .../rpc/interceptor/namespace_rate_limit.go | 12 +- .../interceptor/namespace_rate_limit_test.go | 7 +- common/rpc/interceptor/namespace_validator.go | 65 ++- .../interceptor/namespace_validator_test.go | 58 +- common/rpc/interceptor/nexus/nexus.go | 204 +++++-- common/rpc/interceptor/rate_limit.go | 12 +- common/rpc/interceptor/rate_limit_test.go | 7 +- common/rpc/interceptor/redirection.go | 9 +- common/rpc/interceptor/retry.go | 4 +- .../interceptor/routing_key_interceptor.go | 3 - common/rpc/interceptor/sdk_version.go | 3 +- common/rpc/interceptor/sdk_version_test.go | 2 +- common/rpc/interceptor/slow_request_logger.go | 2 +- .../interceptor/slow_request_logger_test.go | 39 ++ common/rpc/interceptor/telemetry.go | 152 +++--- common/rpc/interceptor/telemetry_test.go | 179 +++--- service/frontend/frontend_interceptors.go | 17 +- service/frontend/fx.go | 69 +-- .../frontend/nexus_completion_http_handler.go | 157 +++--- service/frontend/nexus_forward_interceptor.go | 87 +-- .../nexus_forward_interceptor_test.go | 47 +- service/frontend/nexus_handler.go | 510 +++++++++--------- service/frontend/nexus_handler_test.go | 28 +- .../frontend/nexus_operation_http_handler.go | 96 ++-- temporal/fx.go | 79 ++- temporal/server_option.go | 13 +- temporal/server_options.go | 4 +- tests/nexus_api_validation_test.go | 10 +- tests/nexus_workflow_test.go | 16 +- tests/xdc/nexus_request_forwarding_test.go | 38 +- 39 files changed, 987 insertions(+), 1020 deletions(-) diff --git a/common/authorization/interceptor.go b/common/authorization/interceptor.go index 9131f8deb48..83baa28c4dc 100644 --- a/common/authorization/interceptor.go +++ b/common/authorization/interceptor.go @@ -196,7 +196,7 @@ func (a *Interceptor) InterceptNexus( } a.logger.Error("Authorization internal error with processing nexus request", tag.Error(err)) return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, false), + Err: err, Outcome: "internal_auth_error", } } diff --git a/common/authorization/interceptor_test.go b/common/authorization/interceptor_test.go index 88c6cfa6743..41cffd15d83 100644 --- a/common/authorization/interceptor_test.go +++ b/common/authorization/interceptor_test.go @@ -71,18 +71,19 @@ func TestAuthorizerInterceptorSuite(t *testing.T) { } func (s *authorizerInterceptorSuite) TestInterceptNexus() { + apiName, endpoint := "NexusAPI", "endpoint" input := interceptornexus.NewStartOpInput( "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + interceptornexus.ForwardingInfo{}, + interceptornexus.RequestMetadata{ + APIName: apiName, + EndpointName: endpoint, + }, ) - apiName, endpoint := "NexusAPI", "endpoint" - input.WithRequestMetadata(interceptornexus.RequestMetadata{ - APIName: apiName, - EndpointName: endpoint, - }) expectedTarget := &CallTarget{ APIName: apiName, NexusEndpointName: endpoint, diff --git a/common/rpc/interceptor/caller_info.go b/common/rpc/interceptor/caller_info.go index bcf62d270cf..e199138e83a 100644 --- a/common/rpc/interceptor/caller_info.go +++ b/common/rpc/interceptor/caller_info.go @@ -50,7 +50,7 @@ func (i *CallerInfoInterceptor) InterceptNexus( ctx = PopulateCallerInfo( ctx, in.NamespaceName, - func() string { return nexus.MethodName(in) }, + in.MethodName, ) return next(headers.Propagate(ctx), in) } diff --git a/common/rpc/interceptor/caller_info_test.go b/common/rpc/interceptor/caller_info_test.go index 734214e5071..1196c5635d5 100644 --- a/common/rpc/interceptor/caller_info_test.go +++ b/common/rpc/interceptor/caller_info_test.go @@ -2,6 +2,7 @@ package interceptor import ( "context" + "net/http" "testing" "github.com/nexus-rpc/sdk-go/nexus" @@ -10,6 +11,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/nexus/nexusrpc" interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.uber.org/mock/gomock" "google.golang.org/grpc" @@ -127,6 +129,14 @@ func (s *callerInfoSuite) TestIntercept_CallerName() { } func (s *callerInfoSuite) TestInterceptNexus() { + completeInput, err := interceptornexus.NewCompleteOpInput( + testNamespace, + &nexusrpc.CompletionRequest{HTTPRequest: &http.Request{}}, + nil, + interceptornexus.ForwardingInfo{}, + interceptornexus.RequestMetadata{}, + ) + s.NoError(err) for _, tc := range []struct { name string input interceptornexus.InterceptorInput @@ -135,17 +145,17 @@ func (s *callerInfoSuite) TestInterceptNexus() { }{ { name: "start", - input: interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + input: interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), expectedOrigin: "StartNexusOperation", }, { name: "cancel - preserves background origin", - input: interceptornexus.NewCancelOpInput("s", "o", testNamespace, nexus.CancelOperationOptions{}, "t"), + input: interceptornexus.NewCancelOpInput("s", "o", testNamespace, nexus.CancelOperationOptions{}, "t", interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), callerInfo: headers.SystemBackgroundHighCallerInfo, }, { name: "complete", - input: interceptornexus.NewCompleteOpInput(testNamespace, nil), + input: completeInput, expectedOrigin: "CompleteNexusOperation", }, } { diff --git a/common/rpc/interceptor/concurrent_request_limit.go b/common/rpc/interceptor/concurrent_request_limit.go index f39547894b3..35d5236ec01 100644 --- a/common/rpc/interceptor/concurrent_request_limit.go +++ b/common/rpc/interceptor/concurrent_request_limit.go @@ -12,7 +12,6 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" - commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/quotas/calculator" "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" @@ -129,7 +128,7 @@ func (ni *ConcurrentRequestLimitInterceptor) InterceptNexus( defer cleanup() if err != nil { return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, false), + Err: err, Outcome: "namespace_concurrency_limited", } } diff --git a/common/rpc/interceptor/concurrent_request_limit_test.go b/common/rpc/interceptor/concurrent_request_limit_test.go index f086dd58c0f..c6df21b8518 100644 --- a/common/rpc/interceptor/concurrent_request_limit_test.go +++ b/common/rpc/interceptor/concurrent_request_limit_test.go @@ -152,7 +152,11 @@ func TestConcurrentRequestLimitInterceptor_InterceptNexus(t *testing.T) { dynamicconfig.GetIntPropertyFnFilteredByNamespace(1), map[string]int{"NexusAPI": 1}, ) - input := withAPIName(interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), "NexusAPI") + input := interceptornexus.NewStartOpInput( + "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + interceptornexus.ForwardingInfo{}, + interceptornexus.RequestMetadata{APIName: "NexusAPI"}, + ) ctx := context.Background() @@ -284,23 +288,3 @@ func (h testRequestHandler) Handle(context.Context, any) (any, error) { return nil, nil } - -func withRequestMetadataForTest(in interceptornexus.InterceptorInput, metadata interceptornexus.RequestMetadata) interceptornexus.InterceptorInput { - switch v := in.(type) { - case interceptornexus.StartOpInput: - v.WithRequestMetadata(metadata) - return v - case interceptornexus.CancelOpInput: - v.WithRequestMetadata(metadata) - return v - case interceptornexus.CompleteOpInput: - v.WithRequestMetadata(metadata) - return v - default: - return in - } -} - -func withAPIName(in interceptornexus.InterceptorInput, apiName string) interceptornexus.InterceptorInput { - return withRequestMetadataForTest(in, interceptornexus.RequestMetadata{APIName: apiName}) -} diff --git a/common/rpc/interceptor/context_metadata_interceptor.go b/common/rpc/interceptor/context_metadata_interceptor.go index 64a685fcc5f..d701c6ba46d 100644 --- a/common/rpc/interceptor/context_metadata_interceptor.go +++ b/common/rpc/interceptor/context_metadata_interceptor.go @@ -68,15 +68,7 @@ func (c *ContextMetadataInterceptor) InterceptNexus( in nexus.InterceptorInput, next nexus.HandlerFunc, ) (any, error) { - ctx = contextutil.WithMetadataContext(ctx) - - resp, err := next(ctx, in) - - if c.setTrailer { - c.appendContextMetadataToTrailer(ctx, in.APIName()) - } - - return resp, err + return next(ctx, in) } func (c *ContextMetadataInterceptor) appendContextMetadataToTrailer(ctx context.Context, method string) { diff --git a/common/rpc/interceptor/health.go b/common/rpc/interceptor/health.go index d7f5ae7a0a3..c650b3598ce 100644 --- a/common/rpc/interceptor/health.go +++ b/common/rpc/interceptor/health.go @@ -39,14 +39,12 @@ func (i *HealthInterceptor) Intercept( return handler(ctx, req) } +// InterceptNexus is a no-op as nexus APIs are considered internal func (i *HealthInterceptor) InterceptNexus( ctx context.Context, in nexus.InterceptorInput, next nexus.HandlerFunc, ) (any, error) { - if i.isNotHealthy(in.OperationName()) { - return nil, notHealthyErr - } return next(ctx, in) } diff --git a/common/rpc/interceptor/namespace_handover.go b/common/rpc/interceptor/namespace_handover.go index f3649648720..69822d8d251 100644 --- a/common/rpc/interceptor/namespace_handover.go +++ b/common/rpc/interceptor/namespace_handover.go @@ -92,10 +92,6 @@ func (i *NamespaceHandoverInterceptor) handlesMethod(fullMethod string) bool { return false } -// draft-review: this looks correct, but check in review -// If this is the right way, extract the common logic into a util -// -//nolint:staticcheck func (i *NamespaceHandoverInterceptor) InterceptNexus( ctx context.Context, in nexus.InterceptorInput, diff --git a/common/rpc/interceptor/namespace_rate_limit.go b/common/rpc/interceptor/namespace_rate_limit.go index f04e4e68d33..2c4266d3dbc 100644 --- a/common/rpc/interceptor/namespace_rate_limit.go +++ b/common/rpc/interceptor/namespace_rate_limit.go @@ -12,7 +12,6 @@ import ( "go.temporal.io/server/common/headers" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" - commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/quotas" "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/service/frontend/configs" @@ -118,16 +117,9 @@ func (n *NamespaceRateLimitInterceptorWrapper) InterceptNexus( in nexus.InterceptorInput, next nexus.HandlerFunc, ) (out any, retErr error) { - header, err := nexus.HeaderFromInterceptorInput(in) - if err != nil { + if err := n.ni.Allow(ctx, namespace.Name(in.NamespaceName()), in.APIName(), in.Header()); err != nil { return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), - Outcome: "interceptor_failed", - } - } - if err := n.ni.Allow(namespace.Name(in.NamespaceName()), in.APIName(), header); err != nil { - return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), + Err: err, Outcome: "namespace_rate_limited", } } diff --git a/common/rpc/interceptor/namespace_rate_limit_test.go b/common/rpc/interceptor/namespace_rate_limit_test.go index edbb0604d3f..90a174dc6af 100644 --- a/common/rpc/interceptor/namespace_rate_limit_test.go +++ b/common/rpc/interceptor/namespace_rate_limit_test.go @@ -42,9 +42,8 @@ func (s *namespaceRateLimitInterceptorSuite) TestInterceptNexus() { nextCalled bool expectedOutcome string }{ - {name: "allowed", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), "NexusOperation"), allow: new(true), nextCalled: true}, - {name: "rate limited", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), "NexusOperation"), allow: new(false), expectedOutcome: "namespace_rate_limited"}, - {name: "missing request header", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewCompleteOpInput(testNamespace, nil), "NexusOperation"), expectedOutcome: "interceptor_failed"}, + {name: "allowed", apiName: "NexusOperation", input: interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: new(true), nextCalled: true}, + {name: "rate limited", apiName: "NexusOperation", input: interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: new(false), expectedOutcome: "namespace_rate_limited"}, } { s.Run(tc.name, func() { ctx := context.Background() @@ -53,7 +52,7 @@ func (s *namespaceRateLimitInterceptorSuite) TestInterceptNexus() { } input := tc.input if input == nil { - input = interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) + input = interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}) } nextCalled := false wrapper := NewNamespaceRateLimitInterceptorWrapper(s.newImpl(false)) diff --git a/common/rpc/interceptor/namespace_validator.go b/common/rpc/interceptor/namespace_validator.go index 2c5c83fe47c..492aac6dff4 100644 --- a/common/rpc/interceptor/namespace_validator.go +++ b/common/rpc/interceptor/namespace_validator.go @@ -12,7 +12,6 @@ import ( "go.temporal.io/server/common/api" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/namespace" - commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/tasktoken" "google.golang.org/grpc" @@ -32,11 +31,14 @@ type ( additionalAllowedMethodsDuringHandover map[string]struct{} } - // NamespaceStateValidatorInterceptor contains NamespaceValidatorInterceptor to validate state. - // It is separate from NamespaceValidatorInterceptor to allow both to expose cleaner - // Intercept/InterceptNexus methods that are used as gRPC and Nexus interceptors + // NamespaceStateValidatorInterceptor validates/sets the namespace on a request and enforces + // the namespace name length limit. It is separate from NamespaceValidatorInterceptor to allow + // both to expose cleaner Intercept/InterceptNexus methods that are used as gRPC and Nexus + // interceptors. NamespaceStateValidatorInterceptor struct { - nvi *NamespaceValidatorInterceptor + namespaceRegistry namespace.Registry + tokenSerializer *tasktoken.Serializer + maxNamespaceLength dynamicconfig.IntPropertyFn } ) @@ -117,9 +119,14 @@ func NewNamespaceValidatorInterceptor( } } -func NewNamespaceStateValidatorInterceptor(nvi *NamespaceValidatorInterceptor) *NamespaceStateValidatorInterceptor { +func NewNamespaceStateValidatorInterceptor( + namespaceRegistry namespace.Registry, + maxNamespaceLength dynamicconfig.IntPropertyFn, +) *NamespaceStateValidatorInterceptor { return &NamespaceStateValidatorInterceptor{ - nvi: nvi, + namespaceRegistry: namespaceRegistry, + tokenSerializer: tasktoken.NewSerializer(), + maxNamespaceLength: maxNamespaceLength, } } @@ -129,16 +136,13 @@ func (nsvi *NamespaceStateValidatorInterceptor) Intercept( info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ) (any, error) { - ni := nsvi.nvi - err := ni.setNamespaceIfNotPresent(req) + err := setNamespaceIfNotPresent(nsvi.tokenSerializer, nsvi.namespaceRegistry, req) if err != nil { return nil, err } reqWithNamespace, hasNamespace := req.(NamespaceNameGetter) - if hasNamespace { - if err := ni.ValidateName(reqWithNamespace.GetNamespace()); err != nil { - return nil, err - } + if hasNamespace && len(reqWithNamespace.GetNamespace()) > nsvi.maxNamespaceLength() { + return nil, errNamespaceTooLong } return handler(ctx, req) @@ -149,16 +153,15 @@ func (nsvi *NamespaceStateValidatorInterceptor) InterceptNexus( in nexus.InterceptorInput, next nexus.HandlerFunc, ) (any, error) { - ni := nsvi.nvi ns, err := in.NamespaceEntry() if err != nil { return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, false), + Err: err, Outcome: "interceptor_failed", } } - if err := ni.ValidateName(ns.Info().GetName()); err != nil { - return nil, err + if len(ns.Info().GetName()) > nsvi.maxNamespaceLength() { + return nil, errNamespaceTooLong } return next(ctx, in) @@ -172,17 +175,19 @@ func (ni *NamespaceValidatorInterceptor) ValidateName(ns string) error { return nil } -func (ni *NamespaceValidatorInterceptor) setNamespaceIfNotPresent( +func setNamespaceIfNotPresent( + tokenSerializer *tasktoken.Serializer, + namespaceRegistry namespace.Registry, req any, ) error { switch request := req.(type) { case NamespaceNameGetter: if request.GetNamespace() == "" { - namespaceEntry, err := ni.extractNamespaceFromTaskToken(req) + namespaceEntry, err := extractNamespaceFromTaskToken(tokenSerializer, namespaceRegistry, req) if err != nil { return err } - ni.setNamespace(namespaceEntry, req) + setNamespace(namespaceEntry, req) } return nil default: @@ -190,7 +195,7 @@ func (ni *NamespaceValidatorInterceptor) setNamespaceIfNotPresent( } } -func (ni *NamespaceValidatorInterceptor) setNamespace( +func setNamespace( namespaceEntry *namespace.Namespace, req any, ) { @@ -275,13 +280,13 @@ func (ni *NamespaceValidatorInterceptor) InterceptNexus( namespaceEntry, err := in.NamespaceEntry() if err != nil { return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, false), + Err: err, Outcome: "interceptor_failed", } } if err := ni.ValidateState(namespaceEntry, in.APIName(), in.ForwardingInfo().BusinessID); err != nil { return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, false), + Err: err, Outcome: "invalid_namespace_state", } } @@ -290,7 +295,7 @@ func (ni *NamespaceValidatorInterceptor) InterceptNexus( func (ni *NamespaceValidatorInterceptor) extractNamespace(req any) (*namespace.Namespace, error) { // Token namespace has priority over request namespace. Check it first. - tokenNamespaceEntry, tokenErr := ni.extractNamespaceFromTaskToken(req) + tokenNamespaceEntry, tokenErr := extractNamespaceFromTaskToken(ni.tokenSerializer, ni.namespaceRegistry, req) if tokenErr != nil { return nil, tokenErr } @@ -379,7 +384,11 @@ func (ni *NamespaceValidatorInterceptor) extractNamespaceFromRequest(req any) (* } } -func (ni *NamespaceValidatorInterceptor) extractNamespaceFromTaskToken(req any) (*namespace.Namespace, error) { +func extractNamespaceFromTaskToken( + tokenSerializer *tasktoken.Serializer, + namespaceRegistry namespace.Registry, + req any, +) (*namespace.Namespace, error) { reqWithTaskToken, hasTaskToken := req.(TaskTokenGetter) if !hasTaskToken { return nil, nil @@ -391,13 +400,13 @@ func (ni *NamespaceValidatorInterceptor) extractNamespaceFromTaskToken(req any) var namespaceID namespace.ID // Special case for deprecated RespondQueryTaskCompleted API. if _, ok := req.(*workflowservice.RespondQueryTaskCompletedRequest); ok { - taskToken, err := ni.tokenSerializer.DeserializeQueryTaskToken(taskTokenBytes) + taskToken, err := tokenSerializer.DeserializeQueryTaskToken(taskTokenBytes) if err != nil { return nil, errDeserializingToken } namespaceID = namespace.ID(taskToken.GetNamespaceId()) } else { - taskToken, err := ni.tokenSerializer.Deserialize(taskTokenBytes) + taskToken, err := tokenSerializer.Deserialize(taskTokenBytes) if err != nil { return nil, errDeserializingToken } @@ -407,7 +416,7 @@ func (ni *NamespaceValidatorInterceptor) extractNamespaceFromTaskToken(req any) if namespaceID.IsEmpty() { return nil, errNamespaceNotSet } - return ni.namespaceRegistry.GetNamespaceByID(namespaceID) + return namespaceRegistry.GetNamespaceByID(namespaceID) } func (ni *NamespaceValidatorInterceptor) checkNamespaceMatch(requestNamespace *namespace.Namespace, tokenNamespace *namespace.Namespace) error { diff --git a/common/rpc/interceptor/namespace_validator_test.go b/common/rpc/interceptor/namespace_validator_test.go index d0f1776478b..82b733215e5 100644 --- a/common/rpc/interceptor/namespace_validator_test.go +++ b/common/rpc/interceptor/namespace_validator_test.go @@ -128,8 +128,9 @@ func (s *namespaceValidatorSuite) TestInterceptNexus() { }{ { name: "resolved namespace", - input: withRequestMetadataForTest( - interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + input: interceptornexus.NewStartOpInput( + "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{ APIName: api.NexusServicePrefix + "DispatchNexusTask", NamespaceEntry: namespace.NewNamespaceForTest( @@ -145,8 +146,9 @@ func (s *namespaceValidatorSuite) TestInterceptNexus() { }, { name: "invalid namespace state", - input: withRequestMetadataForTest( - interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + input: interceptornexus.NewStartOpInput( + "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{ APIName: api.NexusServicePrefix + "DispatchNexusTask", NamespaceEntry: namespace.NewNamespaceForTest( @@ -162,9 +164,10 @@ func (s *namespaceValidatorSuite) TestInterceptNexus() { }, { name: "missing namespace", - input: withAPIName( - interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), - "NexusAPI", + input: interceptornexus.NewStartOpInput( + "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + interceptornexus.ForwardingInfo{}, + interceptornexus.RequestMetadata{APIName: "NexusAPI"}, ), expectedOutcome: "interceptor_failed", }, @@ -896,12 +899,10 @@ func (s *namespaceValidatorSuite) Test_Intercept_SearchAttributeRequests() { } func (s *namespaceValidatorSuite) Test_NamespaceValidateIntercept() { - nvi := NewNamespaceValidatorInterceptor( + nnvi := NewNamespaceStateValidatorInterceptor( s.mockRegistry, - dynamicconfig.GetBoolPropertyFn(false), dynamicconfig.GetIntPropertyFn(10), - nil) - nnvi := NewNamespaceStateValidatorInterceptor(nvi) + ) serverInfo := &grpc.UnaryServerInfo{ FullMethod: api.WorkflowServicePrefix + "random", } @@ -966,59 +967,52 @@ func (s *namespaceValidatorSuite) TestSetNamespace() { namespaceEntry, err := namespace.FromPersistentState(detail, factory(detail)) s.NoError(err) - nvi := NewNamespaceValidatorInterceptor( - s.mockRegistry, - dynamicconfig.GetBoolPropertyFn(false), - dynamicconfig.GetIntPropertyFn(10), - nil, - ) - queryReq := &workflowservice.RespondQueryTaskCompletedRequest{} - nvi.setNamespace(namespaceEntry, queryReq) + setNamespace(namespaceEntry, queryReq) s.Equal(namespaceEntryName, queryReq.Namespace) queryReq.Namespace = namespaceRequestName - nvi.setNamespace(namespaceEntry, queryReq) + setNamespace(namespaceEntry, queryReq) s.Equal(namespaceRequestName, queryReq.Namespace) completeWorkflowTaskReq := &workflowservice.RespondWorkflowTaskCompletedRequest{} - nvi.setNamespace(namespaceEntry, completeWorkflowTaskReq) + setNamespace(namespaceEntry, completeWorkflowTaskReq) s.Equal(namespaceEntryName, completeWorkflowTaskReq.Namespace) completeWorkflowTaskReq.Namespace = namespaceRequestName - nvi.setNamespace(namespaceEntry, completeWorkflowTaskReq) + setNamespace(namespaceEntry, completeWorkflowTaskReq) s.Equal(namespaceRequestName, completeWorkflowTaskReq.Namespace) failWorkflowTaskReq := &workflowservice.RespondWorkflowTaskFailedRequest{} - nvi.setNamespace(namespaceEntry, failWorkflowTaskReq) + setNamespace(namespaceEntry, failWorkflowTaskReq) s.Equal(namespaceEntryName, failWorkflowTaskReq.Namespace) failWorkflowTaskReq.Namespace = namespaceRequestName - nvi.setNamespace(namespaceEntry, failWorkflowTaskReq) + setNamespace(namespaceEntry, failWorkflowTaskReq) s.Equal(namespaceRequestName, failWorkflowTaskReq.Namespace) heartbeatActivityTaskReq := &workflowservice.RecordActivityTaskHeartbeatRequest{} - nvi.setNamespace(namespaceEntry, heartbeatActivityTaskReq) + setNamespace(namespaceEntry, heartbeatActivityTaskReq) s.Equal(namespaceEntryName, heartbeatActivityTaskReq.Namespace) heartbeatActivityTaskReq.Namespace = namespaceRequestName - nvi.setNamespace(namespaceEntry, heartbeatActivityTaskReq) + setNamespace(namespaceEntry, heartbeatActivityTaskReq) s.Equal(namespaceRequestName, heartbeatActivityTaskReq.Namespace) cancelActivityTaskReq := &workflowservice.RespondActivityTaskCanceledRequest{} - nvi.setNamespace(namespaceEntry, cancelActivityTaskReq) + setNamespace(namespaceEntry, cancelActivityTaskReq) s.Equal(namespaceEntryName, cancelActivityTaskReq.Namespace) cancelActivityTaskReq.Namespace = namespaceRequestName - nvi.setNamespace(namespaceEntry, cancelActivityTaskReq) + setNamespace(namespaceEntry, cancelActivityTaskReq) s.Equal(namespaceRequestName, cancelActivityTaskReq.Namespace) completeActivityTaskReq := &workflowservice.RespondActivityTaskCompletedRequest{} - nvi.setNamespace(namespaceEntry, completeActivityTaskReq) + setNamespace(namespaceEntry, completeActivityTaskReq) s.Equal(namespaceEntryName, completeActivityTaskReq.Namespace) completeActivityTaskReq.Namespace = namespaceRequestName - nvi.setNamespace(namespaceEntry, completeActivityTaskReq) + setNamespace(namespaceEntry, completeActivityTaskReq) s.Equal(namespaceRequestName, completeActivityTaskReq.Namespace) failActivityTaskReq := &workflowservice.RespondActivityTaskFailedRequest{} - nvi.setNamespace(namespaceEntry, failActivityTaskReq) + setNamespace(namespaceEntry, failActivityTaskReq) s.Equal(namespaceEntryName, failActivityTaskReq.Namespace) failActivityTaskReq.Namespace = namespaceRequestName - nvi.setNamespace(namespaceEntry, failActivityTaskReq) + setNamespace(namespaceEntry, failActivityTaskReq) s.Equal(namespaceRequestName, failActivityTaskReq.Namespace) } diff --git a/common/rpc/interceptor/nexus/nexus.go b/common/rpc/interceptor/nexus/nexus.go index a1f2f6c939e..3158add2e97 100644 --- a/common/rpc/interceptor/nexus/nexus.go +++ b/common/rpc/interceptor/nexus/nexus.go @@ -5,13 +5,30 @@ import ( "errors" "net/http" "slices" + "strings" + "sync" "github.com/nexus-rpc/sdk-go/nexus" + tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/nexus/nexusrpc" ) +const ( + methodNameStartNexusOp = "StartNexusOperation" + methodNameCancelNexusOp = "CancelNexusOperation" + methodNameCompleteNexusOp = "CompleteNexusOperation" + // metric tags + OutcomeInternalError = "internal_error" + OutcomeRequestForwarded = "request_forwarded" + outcomeSyncSuccess = "sync_success" + outcomeAsyncSuccess = "async_success" + outcomeSuccess = "success" + outcomeErrorInternal = "error_internal" +) + type HandlerFunc func(ctx context.Context, in InterceptorInput) (any, error) type Interceptor func(ctx context.Context, in InterceptorInput, next HandlerFunc) (any, error) @@ -24,6 +41,9 @@ type InterceptorInput interface { APIName() string // analogous to the gRPC FullMethod NamespaceEntry() (*namespace.Namespace, error) EndpointName() string + MetricTags() []metrics.Tag + Header() headers.HeaderGetter + MethodName() string sealNexusOp() } @@ -57,29 +77,101 @@ func (t *InterceptorError) Unwrap() error { return t.Err } -// RequestMetadata carries request metadata that is only known once the handler -// has resolved it (e.g. after a namespace registry lookup), and so cannot be supplied -// at InterceptorInput construction time. Set via nexusOpBase.WithRequestMetadata. +// Outcome derives the outcome metric tag value based on the request type and its result +func Outcome(in InterceptorInput, out any, err error) string { + if _, ok := in.(CompleteOpInput); ok { + return completionOutcome(err) + } + if err != nil { + if ie, ok := errors.AsType[*InterceptorError](err); ok && ie.Outcome != "" { + return ie.Outcome + } + return OutcomeInternalError + } + switch out.(type) { + case *nexus.HandlerStartOperationResultSync[any]: + return outcomeSyncSuccess + case *nexus.HandlerStartOperationResultAsync: + return outcomeAsyncSuccess + } + return outcomeSuccess +} + +type outcomeOverrideCtxKey struct{} + +// OutcomeOverride lets an inner interceptor that short-circuits the chain(eg. request forwarder) +// replace the success outcome that would otherwise be derived from the response type +type OutcomeOverride struct { + mu sync.Mutex + value string +} + +func (o *OutcomeOverride) Set(v string) { + if o == nil { + return + } + o.mu.Lock() + defer o.mu.Unlock() + o.value = v +} + +func (o *OutcomeOverride) Get() string { + if o == nil { + return "" + } + o.mu.Lock() + defer o.mu.Unlock() + return o.value +} + +func NewOutcomeOverrideContext(ctx context.Context) (context.Context, *OutcomeOverride) { + override := &OutcomeOverride{} + return context.WithValue(ctx, outcomeOverrideCtxKey{}, override), override +} + +func SetOutcomeOverride(ctx context.Context, v string) { + override, ok := ctx.Value(outcomeOverrideCtxKey{}).(*OutcomeOverride) + if !ok { + return + } + override.Set(v) +} + +func completionOutcome(err error) string { + if err == nil { + return outcomeSuccess + } + if ie, ok := errors.AsType[*InterceptorError](err); ok { + if ie.Outcome != "" { + return ie.Outcome + } + err = ie.Err + } + // retaining behavior + if handlerErr, ok := errors.AsType[*nexus.HandlerError](err); ok { + return "error_" + strings.ToLower(string(handlerErr.Type)) + } + return outcomeErrorInternal +} + +// RequestMetadata carries request metadata resolved by the handler (e.g. after a +// namespace registry lookup) that is supplied alongside the rest of the params at +// InterceptorInput construction time. type RequestMetadata struct { APIName string NamespaceEntry *namespace.Namespace EndpointName string + MetricTags []metrics.Tag // handler-resolved frontend dynamic config for the tags to record } // container for ServiceName(), OperationName(), NamespaceName(), ForwardingInfo(), and // the fields in RequestMetadata. type nexusOpBase struct { - serviceName, operation, namespaceName string - forwardingInfo ForwardingInfo - requestMetadata RequestMetadata -} - -func (b *nexusOpBase) WithForwardingInfo(info ForwardingInfo) { - b.forwardingInfo = info -} - -func (b *nexusOpBase) WithRequestMetadata(metadata RequestMetadata) { - b.requestMetadata = metadata + serviceName, operation, namespaceName, methodName string + header headers.HeaderGetter + // TBD: ForwardingInfo and RequestMetadata could just collapse into nexusOpBase + forwardingInfo ForwardingInfo + requestMetadata RequestMetadata } func (b nexusOpBase) ServiceName() string { @@ -113,41 +205,20 @@ func (b nexusOpBase) EndpointName() string { return b.requestMetadata.EndpointName } -func (nexusOpBase) sealNexusOp() {} +func (b nexusOpBase) MetricTags() []metrics.Tag { + return b.requestMetadata.MetricTags +} -func HeaderFromInterceptorInput(in InterceptorInput) (headers.HeaderGetter, error) { - switch opts := in.(type) { - case StartOpInput: - return opts.StartOperationOptions.Header, nil - case CancelOpInput: - return opts.CancelOperationOptions.Header, nil - case CompleteOpInput: - if opts.CompletionRequest == nil || opts.CompletionRequest.HTTPRequest == nil { - return nil, errors.New("nexus completion request not found") - } - return opts.CompletionRequest.HTTPRequest.Header, nil - default: - return nil, errors.New("unknown Nexus interceptor input") - } +func (b nexusOpBase) Header() headers.HeaderGetter { + return b.header } -// draft-review: verify that these are the "right" methods/names -// TBD: is this different from api.MethodName(in.APIName())? -// -//nolint:staticcheck -func MethodName(in InterceptorInput) string { - switch in.(type) { - case StartOpInput: - return "StartNexusOperation" - case CancelOpInput: - return "CancelNexusOperation" - case CompleteOpInput: - return "CompleteNexusOperation" - default: - return "" - } +func (b nexusOpBase) MethodName() string { + return b.methodName } +func (nexusOpBase) sealNexusOp() {} + type StartOpInput struct { nexusOpBase StartOperationOptions nexus.StartOperationOptions @@ -160,12 +231,18 @@ func NewStartOpInput( namespaceName string, options nexus.StartOperationOptions, input *nexus.LazyValue, + forwardingInfo ForwardingInfo, + requestMetadata RequestMetadata, ) StartOpInput { return StartOpInput{ nexusOpBase: nexusOpBase{ - serviceName: serviceName, - operation: operation, - namespaceName: namespaceName, + serviceName: serviceName, + operation: operation, + namespaceName: namespaceName, + header: options.Header, + methodName: methodNameStartNexusOp, + forwardingInfo: forwardingInfo, + requestMetadata: requestMetadata, }, StartOperationOptions: options, StartOperationInput: input, @@ -184,12 +261,18 @@ func NewCancelOpInput( namespaceName string, options nexus.CancelOperationOptions, cancellationToken string, + forwardingInfo ForwardingInfo, + requestMetadata RequestMetadata, ) CancelOpInput { return CancelOpInput{ nexusOpBase: nexusOpBase{ - serviceName: serviceName, - operation: operation, - namespaceName: namespaceName, + serviceName: serviceName, + operation: operation, + namespaceName: namespaceName, + header: options.Header, + methodName: methodNameCancelNexusOp, + forwardingInfo: forwardingInfo, + requestMetadata: requestMetadata, }, CancelOperationOptions: options, CancellationToken: cancellationToken, @@ -199,21 +282,30 @@ func NewCancelOpInput( type CompleteOpInput struct { nexusOpBase CompletionRequest *nexusrpc.CompletionRequest + Completion *tokenspb.NexusOperationCompletion } -// draft-review: Complete doesnt need servicename/op - verify -// -//nolint:staticcheck func NewCompleteOpInput( namespaceName string, request *nexusrpc.CompletionRequest, -) CompleteOpInput { + completion *tokenspb.NexusOperationCompletion, + forwardingInfo ForwardingInfo, + requestMetadata RequestMetadata, +) (CompleteOpInput, error) { + if request == nil || request.HTTPRequest == nil { + return CompleteOpInput{}, errors.New("nexus completion request not found") + } return CompleteOpInput{ nexusOpBase: nexusOpBase{ - namespaceName: namespaceName, + namespaceName: namespaceName, + header: request.HTTPRequest.Header, + methodName: methodNameCompleteNexusOp, + forwardingInfo: forwardingInfo, + requestMetadata: requestMetadata, }, CompletionRequest: request, - } + Completion: completion, + }, nil } func ChainInterceptors(final HandlerFunc, chain []Interceptor) HandlerFunc { diff --git a/common/rpc/interceptor/rate_limit.go b/common/rpc/interceptor/rate_limit.go index 513f5c9ed1c..e955f62dcd5 100644 --- a/common/rpc/interceptor/rate_limit.go +++ b/common/rpc/interceptor/rate_limit.go @@ -8,7 +8,6 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common/headers" - commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/quotas" "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" @@ -99,16 +98,9 @@ func (i *RateLimitInterceptor) InterceptNexus( in nexus.InterceptorInput, next nexus.HandlerFunc, ) (any, error) { - header, err := nexus.HeaderFromInterceptorInput(in) - if err != nil { + if err := i.Allow(in.APIName(), in.Header()); err != nil { return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), - Outcome: "interceptor_failed", - } - } - if err := i.Allow(in.APIName(), header); err != nil { - return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), + Err: err, Outcome: "global_rate_limited", } } diff --git a/common/rpc/interceptor/rate_limit_test.go b/common/rpc/interceptor/rate_limit_test.go index 445dff1a9ed..a2f1287a1a3 100644 --- a/common/rpc/interceptor/rate_limit_test.go +++ b/common/rpc/interceptor/rate_limit_test.go @@ -37,9 +37,8 @@ func (s *rateLimitInterceptorSuite) TestInterceptNexus() { nextCalled bool expectedOutcome string }{ - {name: "allowed", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil), "NexusOperation"), allow: new(true), nextCalled: true}, - {name: "rate limited", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil), "NexusOperation"), allow: new(false), expectedOutcome: "global_rate_limited"}, - {name: "missing request header", apiName: "NexusOperation", input: withAPIName(interceptornexus.NewCompleteOpInput(testNamespace, nil), "NexusOperation"), expectedOutcome: "interceptor_failed"}, + {name: "allowed", apiName: "NexusOperation", input: interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: new(true), nextCalled: true}, + {name: "rate limited", apiName: "NexusOperation", input: interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: new(false), expectedOutcome: "global_rate_limited"}, } { s.Run(tc.name, func() { ctx := context.Background() @@ -49,7 +48,7 @@ func (s *rateLimitInterceptorSuite) TestInterceptNexus() { } input := tc.input if input == nil { - input = interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil) + input = interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}) } nextCalled := false _, err := interceptor.InterceptNexus( diff --git a/common/rpc/interceptor/redirection.go b/common/rpc/interceptor/redirection.go index 80a1cf01423..1210dec1597 100644 --- a/common/rpc/interceptor/redirection.go +++ b/common/rpc/interceptor/redirection.go @@ -27,7 +27,10 @@ const ( DCRedirectionContextHeaderName = "xdc-redirection" DCRedirectionAPIHeaderName = "xdc-redirection-api" DCRedirectionSourceCellHeaderName = "xdc-redirection-source-cell" - dcRedirectionMetricsPrefix = "DCRedirection" + // DCRedirectionMetricsPrefix prefixes the operation tag on redirection metrics so a + // redirected call is distinguishable from the same operation served locally. Exported + // so the Nexus forwarding interceptor in service/frontend follows the same convention. + DCRedirectionMetricsPrefix = "DCRedirection" ) var ( @@ -287,7 +290,7 @@ func (i *Redirection) handleLocalAPIInvocation( handler grpc.UnaryHandler, methodName string, ) (_ any, retError error) { - scope, startTime := i.BeforeCall(dcRedirectionMetricsPrefix + methodName) + scope, startTime := i.BeforeCall(DCRedirectionMetricsPrefix + methodName) defer func() { i.AfterCall(scope, startTime, i.currentClusterName, "local", retError) }() @@ -307,7 +310,7 @@ func (i *Redirection) handleRedirectAPIInvocation( var targetClusterName = i.currentClusterName var err error - scope, startTime := i.BeforeCall(dcRedirectionMetricsPrefix + methodName) + scope, startTime := i.BeforeCall(DCRedirectionMetricsPrefix + methodName) defer func() { i.AfterCall(scope, startTime, targetClusterName, namespaceName.String(), retError) }() diff --git a/common/rpc/interceptor/retry.go b/common/rpc/interceptor/retry.go index 7cd6d7724b8..c99ff66a570 100644 --- a/common/rpc/interceptor/retry.go +++ b/common/rpc/interceptor/retry.go @@ -44,9 +44,7 @@ func (i *RetryableInterceptor) Intercept( return response, err } -// TBD: evaluate if adding retry is necessary/correct for Nexus -// -//nolint:staticcheck +// InterceptNexus is a no-op as retries are on the caller side func (i *RetryableInterceptor) InterceptNexus( ctx context.Context, in nexus.InterceptorInput, diff --git a/common/rpc/interceptor/routing_key_interceptor.go b/common/rpc/interceptor/routing_key_interceptor.go index e1962ce0e21..909ade6b18e 100644 --- a/common/rpc/interceptor/routing_key_interceptor.go +++ b/common/rpc/interceptor/routing_key_interceptor.go @@ -122,9 +122,6 @@ func (i *RoutingKeyInterceptor) Intercept( return handler(ctx, req) } -// TBD: check if this is needed or if this can also be a passthrough -// -//nolint:staticcheck func (i *RoutingKeyInterceptor) InterceptNexus( ctx context.Context, in nexus.InterceptorInput, diff --git a/common/rpc/interceptor/sdk_version.go b/common/rpc/interceptor/sdk_version.go index 0f5ac1c3196..e473e9ebde5 100644 --- a/common/rpc/interceptor/sdk_version.go +++ b/common/rpc/interceptor/sdk_version.go @@ -5,7 +5,6 @@ import ( "sync" "go.temporal.io/server/common/headers" - commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/versioninfo" "google.golang.org/grpc" @@ -59,7 +58,7 @@ func (vi *SDKVersionInterceptor) InterceptNexus( } if err := vi.versionChecker.ClientSupported(ctx); err != nil { return nil, &nexus.InterceptorError{ - Err: commonnexus.ConvertGRPCError(err, true), + Err: err, Outcome: "unsupported_client", } } diff --git a/common/rpc/interceptor/sdk_version_test.go b/common/rpc/interceptor/sdk_version_test.go index 81fb3eb783c..5e6a9dac0f1 100644 --- a/common/rpc/interceptor/sdk_version_test.go +++ b/common/rpc/interceptor/sdk_version_test.go @@ -102,7 +102,7 @@ func TestSDKVersionInterceptNexus(t *testing.T) { nextCalled := false _, err := interceptor.InterceptNexus( tc.ctx, - interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil), + interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil diff --git a/common/rpc/interceptor/slow_request_logger.go b/common/rpc/interceptor/slow_request_logger.go index d3d0c617463..b07a2b8589e 100644 --- a/common/rpc/interceptor/slow_request_logger.go +++ b/common/rpc/interceptor/slow_request_logger.go @@ -48,7 +48,7 @@ func (i *SlowRequestLoggerInterceptor) InterceptNexus( in nexus.InterceptorInput, next nexus.HandlerFunc, ) (any, error) { - tracker := i.trackSlowRequestFn(in.OperationName(), in) + tracker := i.trackSlowRequestFn(in.APIName(), in) defer tracker() return next(ctx, in) } diff --git a/common/rpc/interceptor/slow_request_logger_test.go b/common/rpc/interceptor/slow_request_logger_test.go index cc1372c3b8a..35a01dfadae 100644 --- a/common/rpc/interceptor/slow_request_logger_test.go +++ b/common/rpc/interceptor/slow_request_logger_test.go @@ -5,12 +5,14 @@ import ( "testing" "time" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/suite" commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/rpc/interceptor" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.uber.org/mock/gomock" "google.golang.org/grpc" ) @@ -96,3 +98,40 @@ func (s *slowRequestLoggerSuite) TestIntercept() { _, err = s.interceptor.Intercept(ctx, nil, info, slowHandler) s.NoError(err) } + +func (s *slowRequestLoggerSuite) TestInterceptNexus() { + ctx := context.Background() + + const nexusDispatchAPIName = "/temporal.api.nexusservice.v1.NexusService/DispatchByNamespaceAndTaskQueue" + + makeNext := func(delay time.Duration) interceptornexus.HandlerFunc { + return func(context.Context, interceptornexus.InterceptorInput) (any, error) { + //nolint:forbidigo // Allow time.Sleep for timeout tests + time.Sleep(delay) + return nil, nil + } + } + fastNext := makeNext(0) + slowNext := makeNext(testThreshold + 1) + + // The operation name here is deliberately not a known API name: the interceptor + // must key off APIName, not OperationName. + input := interceptornexus.NewStartOpInput( + "test-service", + "user-defined-operation", + "namespace-name", + nexus.StartOperationOptions{}, + nil, + interceptornexus.ForwardingInfo{}, + interceptornexus.RequestMetadata{APIName: nexusDispatchAPIName}, + ) + + // Ensure fast requests aren't logged. + _, err := s.interceptor.InterceptNexus(ctx, input, fastNext) + s.Require().NoError(err) + + // Ensure slow requests are logged. + s.logger.EXPECT().Warn(gomock.Eq("Slow gRPC call"), gomock.Any()).Times(1) + _, err = s.interceptor.InterceptNexus(ctx, input, slowNext) + s.Require().NoError(err) +} diff --git a/common/rpc/interceptor/telemetry.go b/common/rpc/interceptor/telemetry.go index ebb918427e0..def718551f1 100644 --- a/common/rpc/interceptor/telemetry.go +++ b/common/rpc/interceptor/telemetry.go @@ -2,9 +2,6 @@ package interceptor import ( "context" - "errors" - "fmt" - "runtime/debug" "strings" "time" @@ -29,8 +26,6 @@ import ( type ( metricsContextKey struct{} - telemetryContextKey struct{} - TelemetryInterceptor struct { namespaceRegistry namespace.Registry metricsHandler metrics.Handler @@ -39,19 +34,6 @@ type ( logAllReqErrors dynamicconfig.BoolPropertyFnWithNamespaceFilter requestErrorHandler ErrorHandler } - - TelemetryContext interface { - MetricsHandler(error) metrics.Handler - MetricsHandlerForInterceptors() metrics.Handler - MetricsLogger() log.Logger - SetMetricsOutcome(string) - // SetFailureSource records which side produced a failure. Only the start/cancel - // handlers return this to the caller; the completion handler discards it. - SetFailureSource(string) - // TBD - check if this is really needed, could remove the error handler interceptor - // HandleRequestError reports a failed request to the shared ErrorHandler. - HandleRequestError(error) - } ) var ( @@ -223,87 +205,83 @@ func AddTelemetryContext(ctx context.Context, metricsHandler metrics.Handler) co return context.WithValue(ctx, metricsCtxKey, metricsHandler) } -// WithTelemetryContext returns a context with telemetry for interceptors that need to -// record their own telemetry - like the forwarder interceptor. -func WithTelemetryContext(ctx context.Context, telemetryContext TelemetryContext) context.Context { - return context.WithValue(ctx, telemetryContextKey{}, telemetryContext) -} - -func TelemetryContextFromContext(ctx context.Context) (TelemetryContext, error) { - telemetryContext, ok := ctx.Value(telemetryContextKey{}).(TelemetryContext) - if !ok { - return nil, errors.New("telemetry context not found") - } - return telemetryContext, nil -} - -// InterceptNexus records request metrics and recovers panics for a Nexus request. -// It runs after auth and redirection, mirroring the gRPC chain, so requests rejected or -// forwarded by those interceptors are not counted here -// It also publishes the metrics context that downstream interceptors read via -// GetMetricsHandlerFromContext. +// InterceptNexus is a no-op as Nexus request telemetry is recorded by +// [*TelemetryInterceptor.InterceptNexusOutermost] func (ti *TelemetryInterceptor) InterceptNexus( ctx context.Context, in nexus.InterceptorInput, next nexus.HandlerFunc, -) (out any, retErr error) { +) (any, error) { + return next(ctx, in) +} - // draft-review: it it not worth splitting the metrics into pre and post forwarder interceptor groups. - // The only "additional" telemetry from fwder is in the case of an actual redirect happening- - // this should anyway be additional metric and get captured in both original and redirected - // clusters as the request did indeed get handled in both places. - // If there is some reason why this should be avoided, then split them such that - // we just "add" the telemetry context as the outermost and then the recorder section - // is added after the authz and fwder interceptors +func (ti *TelemetryInterceptor) InterceptNexusOutermost( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + serviceHandler := ti.metricsHandler.WithTags( + metrics.OperationTag(in.MethodName()), + metrics.NamespaceTag(in.NamespaceName()), + ) + ctx = AddTelemetryContext(ctx, serviceHandler) + metrics.ServiceRequests.With(serviceHandler).Record(1) - telemetryContext, err := TelemetryContextFromContext(ctx) - if err != nil { - return nil, err - } - // required for forwarder interceptor to grab metrics handle when reporting - ctx = metrics.AddMetricsContext(ctx) - ctx = AddTelemetryContext(ctx, telemetryContext.MetricsHandlerForInterceptors()) - interceptorMetricsHandler := telemetryContext.MetricsHandlerForInterceptors() - metrics.ServiceRequests.With(interceptorMetricsHandler).Record(1) + // Installed before calling next so that an inner interceptor that short-circuits the + // chain (e.g. request forwarding) can still override the derived success outcome. + ctx, outcomeOverride := nexus.NewOutcomeOverrideContext(ctx) startTime := time.Now().UTC() - + outcome, failed := nexus.OutcomeInternalError, true defer func() { - reportErr := retErr - if taggedErr, ok := errors.AsType[*nexus.InterceptorError](retErr); ok { - telemetryContext.SetMetricsOutcome(taggedErr.Outcome) - reportErr = taggedErr.Err - } - metricsHandler := telemetryContext.MetricsHandler(reportErr) - switch in.(type) { - case nexus.CompleteOpInput: - metricsHandler.Counter(metrics.NexusCompletionRequests.Name()).Record(1) - metricsHandler.Histogram(metrics.NexusCompletionLatencyHistogram.Name(), metrics.Milliseconds).Record(time.Since(startTime).Milliseconds()) - default: - metrics.NexusRequests.With(metricsHandler).Record(1) - metrics.NexusLatency.With(metricsHandler).Record(time.Since(startTime)) - if reportErr != nil { - metrics.NexusRequestErrors.With(metricsHandler).Record(1) - } - } - ti.RecordLatencyMetrics(ctx, startTime, interceptorMetricsHandler) - telemetryContext.HandleRequestError(reportErr) + ti.RecordLatencyMetrics(ctx, startTime, serviceHandler) + ti.recordNexusRequest(in, startTime, outcome, failed) }() - // recover before recording so that metrics are still recorded in case of a panic - defer func() { - recovered := recover() //nolint:revive - if recovered == nil { - return - } - err, ok := recovered.(error) - if !ok { - err = fmt.Errorf("panic: %v", recovered) + + out, err := next(ctx, in) + outcome, failed = nexus.Outcome(in, out, err), err != nil + + // override outcome if its set - for request forwarding cases. + // error cases are captured by the wrapped InterceptorError + if err == nil { + if override := outcomeOverride.Get(); override != "" { + outcome = override } - telemetryContext.MetricsLogger().Error("Panic captured", tag.SysStackTrace(string(debug.Stack())), tag.Error(err)) - retErr = err - }() + } + return out, err +} - return next(ctx, in) +func (ti *TelemetryInterceptor) recordNexusRequest( + in nexus.InterceptorInput, + startTime time.Time, + outcome string, + failed bool, +) { + if _, ok := in.(nexus.CompleteOpInput); ok { + handler := ti.metricsHandler.WithTags( + metrics.NamespaceTag(in.NamespaceName()), + metrics.OutcomeTag(outcome), + ) + handler.Counter(metrics.NexusCompletionRequests.Name()).Record(1) + handler.Histogram(metrics.NexusCompletionLatencyHistogram.Name(), metrics.Milliseconds). + Record(time.Since(startTime).Milliseconds()) + return + } + + handler := ti.metricsHandler.WithTags( + metrics.NamespaceTag(in.NamespaceName()), + metrics.NexusEndpointTag(in.EndpointName()), + metrics.NexusMethodTag(in.MethodName()), + ) + handler = handler.WithTags(in.MetricTags()...) + // applied last so that a configured tag doesnt shadow the outcome + handler = handler.WithTags(metrics.OutcomeTag(outcome)) + + metrics.NexusRequests.With(handler).Record(1) + metrics.NexusLatency.With(handler).Record(time.Since(startTime)) + if failed { + metrics.NexusRequestErrors.With(handler).Record(1) + } } func (ti *TelemetryInterceptor) RecordLatencyMetrics(ctx context.Context, startTime time.Time, metricsHandler metrics.Handler) { diff --git a/common/rpc/interceptor/telemetry_test.go b/common/rpc/interceptor/telemetry_test.go index 042f88b608e..87af73e7854 100644 --- a/common/rpc/interceptor/telemetry_test.go +++ b/common/rpc/interceptor/telemetry_test.go @@ -22,6 +22,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" serviceerrors "go.temporal.io/server/common/serviceerror" @@ -30,105 +31,125 @@ import ( "google.golang.org/grpc/status" ) -type nexusTelemetryContext struct { - outcome string - handled error - hasHandled bool -} - -func (c *nexusTelemetryContext) MetricsHandler(error) metrics.Handler { - return metrics.NoopMetricsHandler -} - -func (c *nexusTelemetryContext) MetricsHandlerForInterceptors() metrics.Handler { - return metrics.NoopMetricsHandler -} - -func (c *nexusTelemetryContext) MetricsLogger() log.Logger { - return log.NewNoopLogger() -} - -func (c *nexusTelemetryContext) SetMetricsOutcome(outcome string) { - c.outcome = outcome -} - -func (c *nexusTelemetryContext) SetFailureSource(string) {} - -func (c *nexusTelemetryContext) HandleRequestError(err error) { - c.handled = err - c.hasHandled = true -} - -func TestTelemetryInterceptNexus(t *testing.T) { - input := interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil) +func TestTelemetryInterceptNexusOutermost(t *testing.T) { + extraTag := metrics.StringTag("configured", "tag") + input := interceptornexus.NewStartOpInput( + "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + interceptornexus.ForwardingInfo{}, + interceptornexus.RequestMetadata{MetricTags: []metrics.Tag{extraTag}}, + ) for _, tc := range []struct { name string - setContext bool - handler interceptornexus.HandlerFunc + handlerOut any + handlerErr error + setOverride string expectedOutcome string - expectedError error - nextCalled bool - expectHandled bool + expectedErrors int }{ { - name: "regular telemetry capture", - setContext: true, - handler: func(context.Context, interceptornexus.InterceptorInput) (any, error) { - return nil, nil - }, - nextCalled: true, - expectHandled: true, + name: "sync success is derived from the result type", + handlerOut: &nexus.HandlerStartOperationResultSync[any]{}, + expectedOutcome: "sync_success", }, { - name: "missing telemetry context", - handler: func(context.Context, interceptornexus.InterceptorInput) (any, error) { - return nil, nil - }, - expectedError: errors.New("telemetry context not found"), + name: "async success is derived from the result type", + handlerOut: &nexus.HandlerStartOperationResultAsync{}, + expectedOutcome: "async_success", }, { - name: "tagged error", - setContext: true, - handler: func(context.Context, interceptornexus.InterceptorInput) (any, error) { - return nil, &interceptornexus.InterceptorError{Err: errors.New("rejected"), Outcome: "rejected"} - }, + name: "an interceptor's outcome rides on its error", + handlerErr: &interceptornexus.InterceptorError{Err: errors.New("rejected"), Outcome: "rejected"}, expectedOutcome: "rejected", - expectedError: &interceptornexus.InterceptorError{Err: errors.New("rejected"), Outcome: "rejected"}, - nextCalled: true, - expectHandled: true, + expectedErrors: 1, }, { - name: "ensure metrics still captured on panics", - setContext: true, - handler: func(context.Context, interceptornexus.InterceptorInput) (any, error) { - panic("") - }, - expectedError: errors.New("panic: "), - nextCalled: true, - expectHandled: true, + name: "an unclassified error counts as internal", + handlerErr: errors.New("boom"), + expectedOutcome: "internal_error", + expectedErrors: 1, + }, + { + name: "a short-circuiting interceptor overrides the success outcome", + handlerOut: &nexus.HandlerStartOperationResultSync[any]{}, + setOverride: interceptornexus.OutcomeRequestForwarded, + expectedOutcome: "request_forwarded", + }, + { + name: "an error outcome wins over the override", + handlerErr: &interceptornexus.InterceptorError{Err: errors.New("forward failed"), Outcome: "forwarded_request_error"}, + setOverride: interceptornexus.OutcomeRequestForwarded, + expectedOutcome: "forwarded_request_error", + expectedErrors: 1, }, } { t.Run(tc.name, func(t *testing.T) { - ctx := context.Background() - telemetryContext := &nexusTelemetryContext{} - if tc.setContext { - ctx = WithTelemetryContext(ctx, telemetryContext) - } + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + + telemetry := NewTelemetryInterceptor(nil, metricsHandler, log.NewNoopLogger(), nil, nil) nextCalled := false - _, err := (&TelemetryInterceptor{}).InterceptNexus(ctx, input, func(ctx context.Context, input interceptornexus.InterceptorInput) (any, error) { - nextCalled = true - return tc.handler(ctx, input) - }) - require.Equal(t, tc.expectedError, err) - require.Equal(t, tc.nextCalled, nextCalled) - if tc.setContext { - require.Equal(t, tc.expectedOutcome, telemetryContext.outcome) - } - require.Equal(t, tc.expectHandled, telemetryContext.hasHandled) + out, err := telemetry.InterceptNexusOutermost( + context.Background(), + input, + func(ctx context.Context, _ interceptornexus.InterceptorInput) (any, error) { + nextCalled = true + // Downstream interceptors read the published handler from the context. + require.NotNil(t, GetMetricsHandlerFromContext(ctx, log.NewNoopLogger())) + if tc.setOverride != "" { + interceptornexus.SetOutcomeOverride(ctx, tc.setOverride) + } + return tc.handlerOut, tc.handlerErr + }, + ) + require.True(t, nextCalled) + require.Equal(t, tc.handlerOut, out) + require.Equal(t, tc.handlerErr, err) + + snapshot := capture.Snapshot() + namespaceTag := metrics.NamespaceTag(testNamespace) + + outcomeTag := metrics.OutcomeTag(tc.expectedOutcome) + methodTag := metrics.NexusMethodTag("StartNexusOperation") + nexusRequests := snapshot[metrics.NexusRequests.Name()] + require.Len(t, nexusRequests, 1) + require.Equal(t, outcomeTag.Value, nexusRequests[0].Tags[outcomeTag.Key]) + require.Equal(t, methodTag.Value, nexusRequests[0].Tags[methodTag.Key]) + require.Equal(t, namespaceTag.Value, nexusRequests[0].Tags[namespaceTag.Key]) + require.Equal(t, extraTag.Value, nexusRequests[0].Tags[extraTag.Key]) + require.Len(t, snapshot[metrics.NexusLatency.Name()], 1) + require.Len(t, snapshot[metrics.NexusRequestErrors.Name()], tc.expectedErrors) + + requests := snapshot[metrics.ServiceRequests.Name()] + require.Len(t, requests, 1) + require.Equal(t, "StartNexusOperation", requests[0].Tags[metrics.OperationTagName]) + require.Equal(t, namespaceTag.Value, requests[0].Tags[namespaceTag.Key]) + require.Len(t, snapshot[metrics.ServiceLatency.Name()], 1) }) } } +// The shared chain position records nothing; InterceptNexusOutermost is the only recorder. +func TestTelemetryInterceptNexusRecordsNothing(t *testing.T) { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + + telemetry := NewTelemetryInterceptor(nil, metricsHandler, log.NewNoopLogger(), nil, nil) + nextCalled := false + _, err := telemetry.InterceptNexus( + context.Background(), + interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), + func(context.Context, interceptornexus.InterceptorInput) (any, error) { + nextCalled = true + return nil, nil + }, + ) + require.NoError(t, err) + require.True(t, nextCalled) + require.Empty(t, capture.Snapshot()) +} + const ( startWorkflow = "StartWorkflowExecution" executeMultiOps = "ExecuteMultiOperation" diff --git a/service/frontend/frontend_interceptors.go b/service/frontend/frontend_interceptors.go index 48597628ac6..2d5c948eada 100644 --- a/service/frontend/frontend_interceptors.go +++ b/service/frontend/frontend_interceptors.go @@ -33,6 +33,7 @@ type Interceptor interface { type InterceptorsProvider struct { interceptors []Interceptor + nexusTelemetry nexus.Interceptor // required to be first in the Nexus chain retryableInterceptor *interceptor.RetryableInterceptor // required to be last in chain after custom interceptors customGRPCInterceptors []grpc.UnaryServerInterceptor // required for legacy reasons faultGenerator grpcfaults.Generator @@ -54,7 +55,6 @@ func NewInterceptorsProvider( namespaceStateValidatorInterceptor *interceptor.NamespaceStateValidatorInterceptor, namespaceCountLimiterInterceptor *interceptor.ConcurrentRequestLimitInterceptor, namespaceRateLimiterInterceptorWrapper *interceptor.NamespaceRateLimitInterceptorWrapper, - retryableInterceptor *interceptor.RetryableInterceptor, rateLimitInterceptor *interceptor.RateLimitInterceptor, sdkVersionInterceptor *interceptor.SDKVersionInterceptor, callerInfoInterceptor *interceptor.CallerInfoInterceptor, @@ -64,6 +64,7 @@ func NewInterceptorsProvider( customGRPCInterceptors []grpc.UnaryServerInterceptor, customInterceptors []Interceptor, testHooks testhooks.TestHooks, + retryableInterceptor *interceptor.RetryableInterceptor, ) *InterceptorsProvider { interceptors := []Interceptor{ @@ -96,14 +97,15 @@ func NewInterceptorsProvider( return &InterceptorsProvider{ interceptors: interceptors, + nexusTelemetry: telemetryInterceptor.InterceptNexusOutermost, customGRPCInterceptors: customGRPCInterceptors, retryableInterceptor: retryableInterceptor, faultGenerator: grpcfaultstest.NewGenerator(testHooks), } } -func (n *InterceptorsProvider) GetInterceptors() []grpc.UnaryServerInterceptor { - grpcInterceptors := []grpc.UnaryServerInterceptor{} +func (n *InterceptorsProvider) GrpcInterceptors() []grpc.UnaryServerInterceptor { + grpcInterceptors := make([]grpc.UnaryServerInterceptor, 0, len(n.interceptors)+len(n.customGRPCInterceptors)+1) for _, i := range n.interceptors { grpcInterceptors = append(grpcInterceptors, i.Intercept) } @@ -117,8 +119,13 @@ func (n *InterceptorsProvider) GetInterceptors() []grpc.UnaryServerInterceptor { return grpcInterceptors } -func (n *InterceptorsProvider) GetNexusInterceptors() []nexus.Interceptor { - nexusInterceptors := []nexus.Interceptor{} +func (n *InterceptorsProvider) NexusInterceptors() []nexus.Interceptor { + nexusInterceptors := make([]nexus.Interceptor, 0, len(n.interceptors)+2) + // telemetry is the outermost in chain for Nexus requests to allow recording + // all metrics and retain behavior. In the future, gRPC will also move telemetry + // to outermost after an impact evaluation- this will allow gRPC to also capture + // all metrics from authz/redirection related failures as well. + nexusInterceptors = append(nexusInterceptors, n.nexusTelemetry) for _, i := range n.interceptors { nexusInterceptors = append(nexusInterceptors, i.InterceptNexus) } diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 65c748be55a..0d7461c8448 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -135,7 +135,6 @@ var Module = fx.Options( fx.Provide(newNexusForwardingInterceptor), fx.Provide(interceptor.NewNamespaceRateLimitInterceptorWrapper), fx.Provide(NewInterceptorsProvider), - fx.Supply([]Interceptor(nil)), // placeholder for custom unified interceptors thaw will get chained fx.Provide(newNexusCompletionHandler), fx.Provide(NewNexusOperationHTTPHandler), fx.Provide(newNexusCompletionHTTPHandler), @@ -243,31 +242,11 @@ func GrpcServerOptionsProvider( serviceConfig *Config, serviceName primitives.ServiceName, rpcFactory common.RPCFactory, - serviceErrorInterceptor *interceptor.ServiceErrorInterceptor, - namespaceLogInterceptor *interceptor.NamespaceLogInterceptor, - namespaceRateLimiterInterceptor interceptor.NamespaceRateLimitInterceptor, - namespaceCountLimiterInterceptor *interceptor.ConcurrentRequestLimitInterceptor, - namespaceValidatorInterceptor *interceptor.NamespaceValidatorInterceptor, - namespaceStateValidatorInterceptor *interceptor.NamespaceStateValidatorInterceptor, - frontendServiceErrorInterceptor *interceptor.FrontendServiceErrorInterceptor, - namespaceHandoverInterceptor *interceptor.NamespaceHandoverInterceptor, interceptorsProvider *InterceptorsProvider, - businessIDInterceptor *interceptor.RoutingKeyInterceptor, - redirectionInterceptor *interceptor.Redirection, telemetryInterceptor *interceptor.TelemetryInterceptor, - retryableInterceptor *interceptor.RetryableInterceptor, - healthInterceptor *interceptor.HealthInterceptor, - rateLimitInterceptor *interceptor.RateLimitInterceptor, traceStatsHandler telemetry.ServerStatsHandler, metricsStatsHandler metrics.ServerStatsHandler, - sdkVersionInterceptor *interceptor.SDKVersionInterceptor, - callerInfoInterceptor *interceptor.CallerInfoInterceptor, authInterceptor *authorization.Interceptor, - maskInternalErrorDetailsInterceptor *interceptor.MaskInternalErrorDetailsInterceptor, - contextMetadataInterceptor *interceptor.ContextMetadataInterceptor, - slowRequestLoggerInterceptor *interceptor.SlowRequestLoggerInterceptor, - chasmRequestVisibilityInterceptor *chasm.ChasmVisibilityInterceptor, - customInterceptors []grpc.UnaryServerInterceptor, customStreamInterceptors []grpc.StreamServerInterceptor, metricsHandler metrics.Handler, testHooks testhooks.TestHooks, @@ -296,45 +275,8 @@ func GrpcServerOptionsProvider( if err != nil { logger.Fatal("creating gRPC server options failed", tag.Error(err)) } - // unaryInterceptors := []grpc.UnaryServerInterceptor{ - // // Order of interceptors is important - // // Mask error interceptor should be the most outer interceptor since it handle the errors format - // // Service Error Interceptor should be the next most outer interceptor on error handling - // maskInternalErrorDetailsInterceptor.Intercept, - // serviceErrorInterceptor.Intercept, - // frontendServiceErrorInterceptor.Intercept, - // //interceptor.NewFrontendServiceErrorInterceptor(logger), - // // BusinessID interceptor extracts business ID and adds it to context for use, must be before any interceptor that touches namespaces (namespaceValidator, handoverInterceptor) - // businessIDInterceptor.Intercept, - // namespaceStateValidatorInterceptor.Intercept, - // namespaceLogInterceptor.Intercept, // TODO: Deprecate this with a outer custom interceptor - // metrics.NewServerMetricsContextInjectorInterceptor(), // TODO - // authInterceptor.Intercept, - // // Handover interceptor has to above redirection because the request will route to the correct cluster after handover completed. - // // And retry cannot be performed before customInterceptors. - // namespaceHandoverInterceptor.Intercept, - // redirectionInterceptor.Intercept, // TODO, this will have to merge with the nexus frontend interceptor, eval later - // // Telemetry interceptor must be after redirection to ensure metrics are recorded in the correct cluster - // telemetryInterceptor.Intercept, - // healthInterceptor.Intercept, - // namespaceValidatorInterceptor.Intercept, - // namespaceCountLimiterInterceptor.Intercept, - // namespaceRateLimiterInterceptor.Intercept, - // rateLimitInterceptor.Intercept, - // sdkVersionInterceptor.Intercept, - // callerInfoInterceptor.Intercept, - // slowRequestLoggerInterceptor.Intercept, - // chasmRequestVisibilityInterceptor.Intercept, //TODO: this will require nexus interceptor types to be moved out - // contextMetadataInterceptor.Intercept, - // } - // if len(customInterceptors) > 0 { - // // TODO: Deprecate WithChainedFrontendGrpcInterceptors and provide a inner custom interceptor - // unaryInterceptors = append(unaryInterceptors, customInterceptors...) - // } - // // retry interceptor should be the most inner interceptor - // unaryInterceptors = append(unaryInterceptors, retryableInterceptor.Intercept) - - unaryInterceptors := interceptorsProvider.GetInterceptors() + + unaryInterceptors := interceptorsProvider.GrpcInterceptors() streamInterceptor := []grpc.StreamServerInterceptor{ authInterceptor.InterceptStream, @@ -715,9 +657,12 @@ func NamespaceValidatorInterceptorProvider( } func NamespaceStateValidatorInterceptorProvider( - nvi *interceptor.NamespaceValidatorInterceptor, + params NamespaceValidatorInterceptorParams, ) *interceptor.NamespaceStateValidatorInterceptor { - return interceptor.NewNamespaceStateValidatorInterceptor(nvi) + return interceptor.NewNamespaceStateValidatorInterceptor( + params.NamespaceRegistry, + params.ServiceConfig.MaxIDLengthLimit, + ) } func SDKVersionInterceptorProvider() *interceptor.SDKVersionInterceptor { diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index eb5355b4512..52f3ce8f963 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -6,7 +6,6 @@ import ( "net/http" "net/url" "strings" - "time" "github.com/gorilla/mux" "github.com/nexus-rpc/sdk-go/nexus" @@ -49,9 +48,9 @@ type nexusCompletionHandler struct { RequestErrorHandler *interceptor.RequestErrorHandler AuthInterceptor *authorization.Interceptor // required for parsing auth info, not used as an interceptor HTTPTraceProvider commonnexus.HTTPClientTraceProvider - nexusInterceptors []interceptornexus.Interceptor clientVersionChecker headers.VersionChecker preProcessErrorsCounter metrics.CounterIface + chainedHandler interceptornexus.HandlerFunc } type nexusCompletionHTTPHandler struct { @@ -70,10 +69,9 @@ func newNexusCompletionHandler( authInterceptor *authorization.Interceptor, httpTraceProvider commonnexus.HTTPClientTraceProvider, interceptorsProvider *InterceptorsProvider, - customNexusInterceptors []interceptornexus.Interceptor, ) *nexusCompletionHandler { - return &nexusCompletionHandler{ + h := &nexusCompletionHandler{ ClusterMetadata: clusterMetadata, NamespaceRegistry: namespaceRegistry, Logger: log.With(logger, tag.NexusStageCallerInbound), @@ -84,10 +82,11 @@ func newNexusCompletionHandler( RequestErrorHandler: requestErrorHandler, AuthInterceptor: authInterceptor, HTTPTraceProvider: httpTraceProvider, - nexusInterceptors: interceptorsProvider.GetNexusInterceptors(), clientVersionChecker: headers.NewDefaultVersionChecker(), preProcessErrorsCounter: metricsHandler.Counter(metrics.NexusCompletionRequestPreProcessErrors.Name()), } + h.chainedHandler = interceptornexus.ChainInterceptors(h.finalCompleteHandler, interceptorsProvider.NexusInterceptors()) + return h } func newNexusCompletionHTTPHandler(handler *nexusCompletionHandler) *nexusCompletionHTTPHandler { @@ -103,7 +102,6 @@ func newNexusCompletionHTTPHandler(handler *nexusCompletionHandler) *nexusComple // CompleteOperation implements nexus.CompletionHandler. // nolint:revive // (cyclomatic complexity) This function is long but the complexity is justified. func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexusrpc.CompletionRequest) (retErr error) { - startTime := time.Now() token, err := commonnexus.DecodeCallbackToken(r.HTTPRequest.Header.Get(commonnexus.CallbackTokenHeader)) if err != nil { h.Logger.Error("failed to decode callback token", tag.Error(err)) @@ -146,20 +144,17 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus rCtx := &requestContext{ nexusCompletionHandler: h, namespace: ns, - businessID: targetBusinessID, - logger: logger, - metricsHandler: h.MetricsHandler.WithTags(metrics.NamespaceTag(ns.Name().String())), + logger: log.With(h.Logger, tag.WorkflowNamespace(ns.Name().String())), metricsHandlerForInterceptors: h.MetricsHandler.WithTags( metrics.OperationTag(nexusCompletionMethodName), metrics.NamespaceTag(ns.Name().String()), ), - requestStartTime: startTime, } if r.HTTPRequest.Header != nil { rCtx.originalHeaders = r.HTTPRequest.Header.Clone() } ctx = rCtx.augmentContext(ctx, r.HTTPRequest.Header) - defer captureOperationPanic(rCtx.logger, &retErr) + defer finalizeCompletionRequest(rCtx, &retErr) if r.HTTPRequest.URL.Path != commonnexus.PathCompletionCallbackNoIdentifier { nsNameEscaped := commonnexus.RouteCompletionCallback.Deserialize(mux.Vars(r.HTTPRequest)) @@ -182,39 +177,47 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus return err } - interceptorInput := interceptornexus.NewCompleteOpInput(ns.Name().String(), r) - interceptorInput.WithForwardingInfo(interceptornexus.ForwardingInfo{ - OriginalRequestHeaders: rCtx.originalHeaders, - BusinessID: rCtx.businessID, - }) - interceptorInput.WithRequestMetadata(interceptornexus.RequestMetadata{ - APIName: nexusCompletionAPIName, - NamespaceEntry: ns, - }) - finalHandler := func(ctx context.Context, _ interceptornexus.InterceptorInput) (any, error) { - return nil, h.completeOperationRequest(ctx, logger, completion, r, rCtx) - } - _, err = interceptornexus.ChainInterceptors(finalHandler, h.nexusInterceptors)(ctx, interceptorInput) + interceptorInput, err := interceptornexus.NewCompleteOpInput( + ns.Name().String(), + r, + completion, + interceptornexus.ForwardingInfo{ + OriginalRequestHeaders: rCtx.originalHeaders, + BusinessID: targetBusinessID, + }, + interceptornexus.RequestMetadata{ + APIName: nexusCompletionAPIName, + NamespaceEntry: ns, + }, + ) if err != nil { - if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { - return taggedErr.Err - } - return err + logger.Error("invalid nexus completion request", tag.Error(err)) + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid request") } - return nil + ctx = withRequestContext(ctx, rCtx) + _, err = h.chainedHandler(ctx, interceptorInput) + return err } -func (h *nexusCompletionHandler) completeOperationRequest( +func (h *nexusCompletionHandler) finalCompleteHandler( ctx context.Context, - logger log.Logger, - completion *tokenspb.NexusOperationCompletion, - r *nexusrpc.CompletionRequest, - rCtx *requestContext, -) error { + in interceptornexus.InterceptorInput, +) (any, error) { + rCtx, ok := requestContextFromContext(ctx) + if !ok { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid request context for nexus completion") + } + coi, ok := in.(interceptornexus.CompleteOpInput) + if !ok { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid request for nexus complete operation") + } + logger := rCtx.logger + completion := coi.Completion + r := coi.CompletionRequest ns := rCtx.namespace tokenLimit := h.Config.MaxNexusOperationTokenLength(ns.Name().String()) if len(r.OperationToken) > tokenLimit { - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "operation token length exceeds allowed limit (%d/%d)", len(r.OperationToken), tokenLimit) + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "operation token length exceeds allowed limit (%d/%d)", len(r.OperationToken), tokenLimit) } links := commonnexus.ConvertNexusLinksToProtoLinks(r.Links, logger) @@ -227,31 +230,31 @@ func (h *nexusCompletionHandler) completeOperationRequest( var result *commonpb.Payload if err := r.Result.Consume(&result); err != nil { logger.Error("cannot deserialize payload from completion result", tag.Error(err)) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid result content") + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid result content") } if result.Size() > h.Config.BlobSizeLimitError(ns.Name().String()) { - logger.Error("payload size exceeds error limit for Nexus CompleteOperation request") - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "result exceeds size limit") + logger.Error("payload size exceeds error limit for Nexus CompleteOperation request", tag.WorkflowNamespace(ns.Name().String())) + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "result exceeds size limit") } successPayload = result default: // The Nexus SDK ensures this never happens but just in case... logger.Error("invalid operation state in completion request", tag.String("state", string(r.State))) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid completion state") + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid completion state") } err := h.completeOperation(ctx, logger, completion, successPayload, r, links, h.Config.EnableChasm(ns.Name().String())) if err == nil { - return nil + return nil, nil } logger.Error("failed to process nexus completion request", tag.Error(err)) if _, ok := errors.AsType[*serviceerror.NamespaceNotActive](err); ok { - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive") + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive") } if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { - return commonnexus.ConvertGRPCError(err, true) + return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "error_not_found"} } - return commonnexus.ConvertGRPCError(err, false) + return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "error_internal"} } // completeOperation dispatches the completion to the framework named by its @@ -420,17 +423,27 @@ func (h *nexusCompletionHTTPHandler) RegisterRoutes(r *mux.Router) { type requestContext struct { *nexusCompletionHandler logger log.Logger - metricsHandler metrics.Handler metricsHandlerForInterceptors metrics.Handler - namespace *namespace.Namespace - businessID string - requestStartTime time.Time - outcomeTag metrics.Tag + namespace *namespace.Namespace // required for reporting via handleRequestError originalHeaders http.Header } +// Key to extract a *requestContext from a context.Context. +type requestContextKey struct{} + +func withRequestContext(ctx context.Context, rCtx *requestContext) context.Context { + if rCtx == nil { + return ctx + } + return context.WithValue(ctx, requestContextKey{}, rCtx) +} + +func requestContextFromContext(ctx context.Context) (*requestContext, bool) { + rCtx, ok := ctx.Value(requestContextKey{}).(*requestContext) + return rCtx, ok +} + func (c *requestContext) augmentContext(ctx context.Context, header http.Header) context.Context { - ctx = interceptor.WithTelemetryContext(ctx, c) if userAgent := header.Get(headerUserAgent); userAgent != "" { // Preserve original strict behavior: only process if exactly one delimiter present. if strings.Count(userAgent, clientNameVersionDelim) == 1 { @@ -449,50 +462,34 @@ func (c *requestContext) augmentContext(ctx context.Context, header http.Header) return ctx } -func (c *requestContext) MetricsHandler(err error) metrics.Handler { - if c.outcomeTag.Key != "" { - return c.metricsHandler.WithTags(c.outcomeTag) - } - if err == nil { - return c.metricsHandler.WithTags(metrics.OutcomeTag("success")) - } - if handlerErr, ok := errors.AsType[*nexus.HandlerError](err); ok { - return c.metricsHandler.WithTags(metrics.OutcomeTag("error_" + strings.ToLower(string(handlerErr.Type)))) - } - return c.metricsHandler.WithTags(metrics.OutcomeTag("error_internal")) -} - -func (c *requestContext) MetricsHandlerForInterceptors() metrics.Handler { - return c.metricsHandlerForInterceptors -} - -func (c *requestContext) MetricsLogger() log.Logger { - return c.logger -} - -func (c *requestContext) SetMetricsOutcome(outcome string) { - c.outcomeTag = metrics.OutcomeTag(outcome) -} - -// no-op for completion as it doesnt report back via headers -func (c *requestContext) SetFailureSource(string) {} - -func (c *requestContext) HandleRequestError(err error) { +func (c *requestContext) handleRequestError(err error) { if err == nil { return } + if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { + err = taggedErr.Err + } c.RequestErrorHandler.HandleError( // The request is only read to extract workflow log tags, which is keyed off the // gRPC full method. Nexus has none, so it is never used. nil, "", c.metricsHandlerForInterceptors, - []tag.Tag{tag.Operation(nexusCompletionMethodNameForMetrics), tag.WorkflowNamespace(c.namespace.Name().String())}, + []tag.Tag{tag.Operation(nexusCompletionMethodName), tag.WorkflowNamespace(c.namespace.Name().String())}, err, c.namespace.Name(), ) } +// finalizeCompletionRequest is the single deferred step for a Nexus completion request: capture a +// panic into errPtr, log/classify the (still raw) resulting error, then sanitize it for the +// response. Order matters and must not be split back into separate defers. +func finalizeCompletionRequest(rCtx *requestContext, errPtr *error) { + captureOperationPanic(rCtx.logger, errPtr) + rCtx.handleRequestError(*errPtr) + *errPtr = convertInterceptorError(*errPtr) +} + // enrich context with authInfo func (c *requestContext) parseTLSAndAuthInfo(ctx context.Context, request *nexusrpc.CompletionRequest) (context.Context, error) { var tlsInfo *credentials.TLSInfo diff --git a/service/frontend/nexus_forward_interceptor.go b/service/frontend/nexus_forward_interceptor.go index c9c8936f7fa..b7a86ec0734 100644 --- a/service/frontend/nexus_forward_interceptor.go +++ b/service/frontend/nexus_forward_interceptor.go @@ -59,13 +59,7 @@ func (i *nexusForwardingInterceptor) InterceptNexus( next interceptornexus.HandlerFunc, ) (out any, retErr error) { info := in.ForwardingInfo() - header, err := interceptornexus.HeaderFromInterceptorInput(in) - if err != nil { - return nil, &interceptornexus.InterceptorError{ - Err: err, - Outcome: "interceptor_failed", - } - } + header := in.Header() namespaceEntry, err := in.NamespaceEntry() if err != nil { return nil, &interceptornexus.InterceptorError{ @@ -85,16 +79,13 @@ func (i *nexusForwardingInterceptor) InterceptNexus( } } - telemetryContext, err := interceptor.TelemetryContextFromContext(ctx) - if err != nil { - return nil, &interceptornexus.InterceptorError{ - Err: err, - Outcome: "interceptor_failed", - } - } - telemetryContext.SetMetricsOutcome("request_forwarded") + interceptornexus.SetOutcomeOverride(ctx, interceptornexus.OutcomeRequestForwarded) - metricsHandler, forwardStartTime := i.redirectionInterceptor.BeforeCall(interceptornexus.MethodName(in)) + // this is the user-facing operation identity, and the DCRedirection prefix + // matches the convention the gRPC redirection path uses for the same metrics. + metricsHandler, forwardStartTime := i.redirectionInterceptor.BeforeCall( + interceptor.DCRedirectionMetricsPrefix + in.MethodName(), + ) defer func() { redirectionErr := retErr if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](retErr); ok { @@ -105,11 +96,11 @@ func (i *nexusForwardingInterceptor) InterceptNexus( switch request := in.(type) { case interceptornexus.StartOpInput: - out, retErr = i.forwardStartOperation(ctx, request, info, namespaceEntry, targetCluster, telemetryContext) + out, retErr = i.forwardStartOperation(ctx, request, info, namespaceEntry, targetCluster) case interceptornexus.CancelOpInput: - retErr = i.forwardCancelOperation(ctx, request, info, namespaceEntry, targetCluster, telemetryContext) + retErr = i.forwardCancelOperation(ctx, request, info, namespaceEntry, targetCluster) case interceptornexus.CompleteOpInput: - retErr = i.forwardCompleteOperation(ctx, request, info, namespaceEntry, targetCluster, telemetryContext) + retErr = i.forwardCompleteOperation(ctx, request, info, namespaceEntry, targetCluster) default: return nil, &interceptornexus.InterceptorError{ Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "forwarding failed, unknown operation type"), @@ -139,18 +130,22 @@ func (i *nexusForwardingInterceptor) forwardStartOperation( info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, - telemetryContext interceptor.TelemetryContext, ) (any, error) { + logger := log.With( + i.logger, + tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), + tag.TargetCluster(targetCluster), + ) request.StartOperationOptions.Header[interceptor.DCRedirectionAPIHeaderName] = "true" request.StartOperationOptions.Header[interceptor.DCRedirectionSourceCellHeaderName] = i.clusterMetadata.GetCurrentClusterName() - client, err := i.nexusClientForActiveCluster(request.ServiceName(), info, namespaceEntry, targetCluster, telemetryContext) + client, err := i.nexusClientForActiveCluster(ctx, request.ServiceName(), info, namespaceEntry, targetCluster) if err != nil { return nil, err } ctx = i.withForwardingTrace(ctx, "StartNexusOperation", request.OperationName(), request.StartOperationOptions.RequestID, info, namespaceEntry, targetCluster) response, err := client.StartOperation(ctx, request.OperationName(), request.StartOperationInput.Reader, request.StartOperationOptions) if err != nil { - i.logger.Error("received error from remote cluster for forwarded Nexus start operation request", tag.Error(err)) + logger.Error("received error from remote cluster for forwarded Nexus start operation request", tag.Error(err)) return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error"} } if response.Successful != nil { @@ -165,22 +160,26 @@ func (i *nexusForwardingInterceptor) forwardCancelOperation( info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, - telemetryContext interceptor.TelemetryContext, ) error { + logger := log.With( + i.logger, + tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), + tag.TargetCluster(targetCluster), + ) request.CancelOperationOptions.Header[interceptor.DCRedirectionAPIHeaderName] = "true" request.CancelOperationOptions.Header[interceptor.DCRedirectionSourceCellHeaderName] = i.clusterMetadata.GetCurrentClusterName() - client, err := i.nexusClientForActiveCluster(request.ServiceName(), info, namespaceEntry, targetCluster, telemetryContext) + client, err := i.nexusClientForActiveCluster(ctx, request.ServiceName(), info, namespaceEntry, targetCluster) if err != nil { return err } handle, err := client.NewOperationHandle(request.OperationName(), request.CancellationToken) if err != nil { - i.logger.Warn("invalid Nexus cancel operation", tag.Error(err)) + logger.Warn("invalid Nexus cancel operation", tag.Error(err)) return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation") } ctx = i.withForwardingTrace(ctx, "CancelNexusOperation", request.OperationName(), "", info, namespaceEntry, targetCluster) if err := handle.Cancel(ctx, request.CancelOperationOptions); err != nil { - i.logger.Error("received error from remote cluster for forwarded Nexus cancel operation request", tag.Error(err)) + logger.Error("received error from remote cluster for forwarded Nexus cancel operation request", tag.Error(err)) return &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error"} } return nil @@ -192,16 +191,20 @@ func (i *nexusForwardingInterceptor) forwardCompleteOperation( info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, - telemetryContext interceptor.TelemetryContext, ) error { + logger := log.With( + i.logger, + tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), + tag.TargetCluster(targetCluster), + ) client, err := i.forwardingClients.Get(targetCluster) if err != nil { - i.logger.Error("unable to get HTTP client for forward request", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) + logger.Error("unable to get HTTP client for forward request", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) return &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} } forwardURL, err := url.JoinPath(client.BaseURL(), commonnexus.RouteCompletionCallback.Path(namespaceEntry.Name().String())) if err != nil { - i.logger.Error("failed to construct forwarding request URL", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.TargetCluster(targetCluster)) + logger.Error("failed to construct forwarding request URL", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.TargetCluster(targetCluster)) return &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} } request.CompletionRequest.HTTPRequest.Header.Set(interceptor.DCRedirectionAPIHeaderName, "true") @@ -214,7 +217,8 @@ func (i *nexusForwardingInterceptor) forwardCompleteOperation( } ctx = i.withForwardingTrace(ctx, "CompleteNexusOperation", "", "", info, namespaceEntry, targetCluster) err = nexusrpc.NewCompletionHTTPClient(nexusrpc.CompletionHTTPClientOptions{ - HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: client, originalRequestHeaders: info.OriginalRequestHeaders, telemetryContext: telemetryContext}).Do, + // completions dont report a failure source back to the caller through headers + HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: client, originalRequestHeaders: info.OriginalRequestHeaders}).Do, }).CompleteOperation(ctx, forwardURL, completion) if err != nil { return &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error"} @@ -234,15 +238,24 @@ func completeOperationOptions(request *nexusrpc.CompletionRequest) (nexusrpc.Com } func (i *nexusForwardingInterceptor) nexusClientForActiveCluster( + ctx context.Context, service string, info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, - telemetryContext interceptor.TelemetryContext, ) (*nexusrpc.HTTPClient, error) { + var setFailureSource func(string) // required for setting the source in case of a failure + if oc, ok := operationContextFromContext(ctx); ok { + setFailureSource = oc.setFailureSource + } + logger := log.With( + i.logger, + tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), + tag.TargetCluster(targetCluster), + ) httpClient, err := i.forwardingClients.Get(targetCluster) if err != nil { - i.logger.Error("failed to forward Nexus request: error creating HTTP client", tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) + logger.Error("failed to forward Nexus request: error creating HTTP client", tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) return nil, &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} } var baseURL string @@ -252,11 +265,11 @@ func (i *nexusForwardingInterceptor) nexusClientForActiveCluster( baseURL, err = url.JoinPath(httpClient.BaseURL(), commonnexus.RouteDispatchNexusTaskByNamespaceAndTaskQueue.Path(commonnexus.NamespaceAndTaskQueue{Namespace: namespaceEntry.Name().String(), TaskQueue: info.TaskQueue})) } if err != nil { - i.logger.Error("failed to forward Nexus request: error constructing ServiceBaseURL", tag.URL(httpClient.BaseURL()), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.WorkflowTaskQueueName(info.TaskQueue), tag.Error(err)) + logger.Error("failed to forward Nexus request: error constructing ServiceBaseURL", tag.URL(httpClient.BaseURL()), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.WorkflowTaskQueueName(info.TaskQueue), tag.Error(err)) return nil, &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} } return nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ - HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: httpClient, originalRequestHeaders: info.OriginalRequestHeaders, telemetryContext: telemetryContext}).Do, + HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: httpClient, originalRequestHeaders: info.OriginalRequestHeaders, setFailureSource: setFailureSource}).Do, BaseURL: baseURL, Service: service, }) @@ -293,7 +306,7 @@ func (i *nexusForwardingInterceptor) withForwardingTrace( type nexusForwardingHTTPHeaderWrapper struct { client *common.FrontendHTTPClient originalRequestHeaders http.Header - telemetryContext interceptor.TelemetryContext + setFailureSource func(string) } func (f *nexusForwardingHTTPHeaderWrapper) Do(request *http.Request) (*http.Response, error) { @@ -308,8 +321,8 @@ func (f *nexusForwardingHTTPHeaderWrapper) Do(request *http.Request) (*http.Resp return nil, err } - if source := response.Header.Get(commonnexus.FailureSourceHeaderName); source != "" && f.telemetryContext != nil { - f.telemetryContext.SetFailureSource(source) + if source := response.Header.Get(commonnexus.FailureSourceHeaderName); source != "" && f.setFailureSource != nil { + f.setFailureSource(source) } return response, nil } diff --git a/service/frontend/nexus_forward_interceptor_test.go b/service/frontend/nexus_forward_interceptor_test.go index 01a194c6f20..fc116ef36fe 100644 --- a/service/frontend/nexus_forward_interceptor_test.go +++ b/service/frontend/nexus_forward_interceptor_test.go @@ -52,16 +52,17 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { Scheme: "http", }, }} - input := interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{ + options := nexus.StartOperationOptions{ Header: nexus.Header{"X-Request": "request"}, - }, nexus.NewLazyValue(nexus.DefaultSerializer(), &nexus.Reader{ + } + requestInput := nexus.NewLazyValue(nexus.DefaultSerializer(), &nexus.Reader{ ReadCloser: io.NopCloser(bytes.NewBufferString(`"input"`)), Header: nexus.Header{"type": "json"}, - })) - input.WithForwardingInfo(interceptornexus.ForwardingInfo{ + }) + forwardingInfo := interceptornexus.ForwardingInfo{ OriginalRequestHeaders: http.Header{"X-Original": {"original"}}, TaskQueue: "task-queue", - }) + } for _, tc := range []struct { name string @@ -139,14 +140,16 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { NexusForwardRequestUseEndpoint: dynamicconfig.GetBoolPropertyFn(false), }, } - input.WithRequestMetadata(interceptornexus.RequestMetadata{ - NamespaceEntry: tc.namespace, - }) - ctx := interceptor.WithTelemetryContext(context.Background(), &forwardingTelemetryContext{}) + in := interceptornexus.NewStartOpInput( + "s", "o", testNamespace, options, requestInput, + forwardingInfo, + interceptornexus.RequestMetadata{NamespaceEntry: tc.namespace}, + ) + ctx := context.Background() nextCalled := false result, err := forwarder.InterceptNexus( ctx, - input, + in, func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return requestHandledLocally, nil @@ -175,30 +178,6 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { } } -type forwardingTelemetryContext struct { - failureSource string -} - -func (*forwardingTelemetryContext) MetricsHandler(error) metrics.Handler { - return metrics.NoopMetricsHandler -} - -func (*forwardingTelemetryContext) MetricsHandlerForInterceptors() metrics.Handler { - return metrics.NoopMetricsHandler -} - -func (*forwardingTelemetryContext) MetricsLogger() log.Logger { - return log.NewNoopLogger() -} - -func (*forwardingTelemetryContext) SetMetricsOutcome(string) {} - -func (c *forwardingTelemetryContext) SetFailureSource(source string) { - c.failureSource = source -} - -func (*forwardingTelemetryContext) HandleRequestError(error) {} - type testFrontendHTTPClientCache struct { clients map[string]*common.FrontendHTTPClient } diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index b14dab872b0..588d2be718e 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net/http" - "net/http/httptrace" "net/url" "regexp" "runtime/debug" @@ -14,13 +13,13 @@ import ( "time" "github.com/nexus-rpc/sdk-go/nexus" + "go.opentelemetry.io/otel/trace" enumspb "go.temporal.io/api/enums/v1" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/server/api/matchingservice/v1" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" - "go.temporal.io/server/common/authorization" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" @@ -47,21 +46,16 @@ const ( type nexusContext struct { // Whether to use the new Temporal failure responses path. // Set from the incoming nexus request's "temporal-nexus-failure-support" header. - callerFailureSupport bool - requestStartTime time.Time - apiName string - namespaceName string - taskQueue string - endpointName string - endpointID string - claims *authorization.Claims - namespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor - namespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor - namespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor - rateLimitInterceptor *interceptor.RateLimitInterceptor - responseHeaders map[string]string - responseHeadersMutex sync.Mutex - originalRequestHeaders http.Header // Original HTTP request headers to be used for forwarded requests. + callerFailureSupport bool + requestStartTime time.Time + apiName string + namespaceName string + taskQueue string + endpointName string + endpointID string + responseHeaders map[string]string + responseHeadersMutex sync.Mutex + originalRequestHeaders http.Header // Original HTTP request headers to be used for forwarded requests. } // Context for a specific Nexus operation, includes a resolved namespace, and a bound metrics handler and logger. @@ -76,10 +70,8 @@ type operationContext struct { metricsHandler metrics.Handler logger log.Logger clientVersionChecker headers.VersionChecker - telemetryInterceptor *interceptor.TelemetryInterceptor requestErrorHandler *interceptor.RequestErrorHandler headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] - metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig] } func (c *operationContext) matchingRequest(req *nexuspb.Request) *matchingservice.DispatchNexusTaskRequest { @@ -91,8 +83,16 @@ func (c *operationContext) matchingRequest(req *nexuspb.Request) *matchingservic } } +func (c *operationContext) annotateServerSpan(ctx context.Context, service, operation, requestID string) { + nexusrpc.AnnotateServerSpan(trace.SpanFromContext(ctx), nexusrpc.ServerSpanAttributes{ + Endpoint: c.endpointName, + Service: service, + Operation: operation, + RequestID: requestID, + }) +} + func (c *operationContext) augmentContext(ctx context.Context, header nexus.Header) context.Context { - ctx = interceptor.WithTelemetryContext(ctx, c) if userAgent, ok := header[headerUserAgent]; ok { // Use SplitN for efficiency but enforce exactly one delimiter to preserve the // original (pre-SplitN) strictness where additional delimiters cause us to ignore @@ -113,31 +113,13 @@ func (c *operationContext) augmentContext(ctx context.Context, header nexus.Head return ctx } -func (c *operationContext) MetricsHandler(_ error) metrics.Handler { - // start/cancel handlers already have the error, just return the handler - return c.metricsHandler -} - -func (c *operationContext) MetricsHandlerForInterceptors() metrics.Handler { - return c.metricsHandlerForInterceptors -} - -func (c *operationContext) MetricsLogger() log.Logger { - return c.logger -} - -func (c *operationContext) SetMetricsOutcome(outcome string) { - c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag(outcome)) -} - -func (c *operationContext) SetFailureSource(source string) { - c.setFailureSource(source) -} - -func (c *operationContext) HandleRequestError(err error) { +func (c *operationContext) handleRequestError(err error) { if err == nil { return } + if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { + err = taggedErr.Err + } source, ok := c.responseHeaders[commonnexus.FailureSourceHeaderName] if !ok || source == commonnexus.FailureSourceWorker { return @@ -154,8 +136,7 @@ func (c *operationContext) HandleRequestError(err error) { ) } -// required as operations might panic before the interceptor chain is -// invoked and panics are handled by the outermost telemetry interceptor +// required as operations might panic before the interceptor chain is invoked func captureOperationPanic(logger log.Logger, errPtr *error) { recovered := recover() //nolint:revive if recovered == nil { @@ -169,13 +150,37 @@ func captureOperationPanic(logger log.Logger, errPtr *error) { *errPtr = err } -func (c *operationContext) sanitizeRequestHeaders(request *matchingservice.DispatchNexusTaskRequest) { +// convertInterceptorError converts the error returned by the interceptor chain into the sanitized +// form returned to the Nexus caller, hiding internal error detail. Interceptors intentionally leave +// InterceptorError.Err raw so the boundary can log/classify the full original error via +// [*operationContext.handleRequestError] before this runs and replaces it for the response. +func convertInterceptorError(err error) error { + if err == nil { + return nil + } + if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { + // always convert error to omit exposing details to end callers + return commonnexus.ConvertGRPCError(taggedErr.Err, false) + } + return err +} + +// finalizeOperationRequest is the single deferred step for a Nexus start/cancel operation: capture +// a panic into errPtr, log/classify the (still raw) resulting error, then sanitize it for the +// response. Order matters and must not be split back into separate defers. +func finalizeOperationRequest(oc *operationContext, errPtr *error) { + captureOperationPanic(oc.logger, errPtr) + oc.handleRequestError(*errPtr) + *errPtr = convertInterceptorError(*errPtr) +} + +func (h *nexusHandler) sanitizeRequestHeaders(request *matchingservice.DispatchNexusTaskRequest) { if request.GetRequest().GetHeader() == nil { return } sanitizedHeaders := make(map[string]string, len(request.Request.Header)) - headersBlacklist := c.headersBlacklist() + headersBlacklist := h.headersBlacklist() for name, value := range request.Request.Header { if !headersBlacklist.MatchString(name) { sanitizedHeaders[name] = value @@ -184,29 +189,6 @@ func (c *operationContext) sanitizeRequestHeaders(request *matchingservice.Dispa request.Request.Header = sanitizedHeaders } -// enrichNexusOperationMetrics enhances metrics with additional Nexus operation context based on configuration. -func (c *operationContext) enrichNexusOperationMetrics(service, operation string, requestHeader nexus.Header) { - conf := c.metricTagConfig() - - var tags []metrics.Tag - - if conf.IncludeServiceTag { - tags = append(tags, metrics.NexusServiceTag(service)) - } - - if conf.IncludeOperationTag { - tags = append(tags, metrics.NexusOperationTag(operation)) - } - - for _, mapping := range conf.HeaderTagMappings { - tags = append(tags, metrics.StringTag(mapping.TargetTag, requestHeader.Get(mapping.SourceHeader))) - } - - if len(tags) > 0 { - c.metricsHandler = c.metricsHandler.WithTags(tags...) - } -} - // enrichNexusOperationLogs adds Nexus operation context to the handler-side logger. func (c *operationContext) enrichNexusOperationLogs(service, operation, requestID string) { tags := []tag.Tag{ @@ -223,6 +205,20 @@ func (c *operationContext) enrichNexusOperationLogs(service, operation, requestI // Key to extract a nexusContext object from a context.Context. type nexusContextKey struct{} +type operationContextKey struct{} + +func withOperationContext(ctx context.Context, oc *operationContext) context.Context { + if oc == nil { + return ctx + } + return context.WithValue(ctx, operationContextKey{}, oc) +} + +func operationContextFromContext(ctx context.Context) (*operationContext, bool) { + oc, ok := ctx.Value(operationContextKey{}).(*operationContext) + return oc, ok +} + // A Nexus Handler implementation. // Dispatches Nexus requests as Nexus tasks to workers via matching. type nexusHandler struct { @@ -238,7 +234,56 @@ type nexusHandler struct { useForwardByEndpoint dynamicconfig.BoolPropertyFn metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig] httpTraceProvider commonnexus.HTTPClientTraceProvider - nexusInterceptors []interceptornexus.Interceptor + chainedHandler interceptornexus.HandlerFunc +} + +func newNexusHandler( + logger log.Logger, + metricsHandler metrics.Handler, + clusterMetadata cluster.Metadata, + namespaceRegistry namespace.Registry, + matchingClient matchingservice.MatchingServiceClient, + requestErrorHandler *interceptor.RequestErrorHandler, + payloadSizeLimit dynamicconfig.IntPropertyFnWithNamespaceFilter, + headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp], + useForwardByEndpoint dynamicconfig.BoolPropertyFn, + metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig], + httpTraceProvider commonnexus.HTTPClientTraceProvider, + nexusInterceptors []interceptornexus.Interceptor, +) *nexusHandler { + h := &nexusHandler{ + logger: logger, + metricsHandler: metricsHandler, + clusterMetadata: clusterMetadata, + namespaceRegistry: namespaceRegistry, + matchingClient: matchingClient, + requestErrorHandler: requestErrorHandler, + payloadSizeLimit: payloadSizeLimit, + headersBlacklist: headersBlacklist, + useForwardByEndpoint: useForwardByEndpoint, + metricTagConfig: metricTagConfig, + httpTraceProvider: httpTraceProvider, + } + h.chainedHandler = interceptornexus.ChainInterceptors(h.finalHandler, nexusInterceptors) + return h +} + +// nexusMetricTags resolves the operator-configurable tags for this request's Nexus metrics. Only the +// frontend can read the configuration, so the tags travel to the telemetry interceptor as request +// metadata rather than being built where they are recorded. +func (h *nexusHandler) nexusMetricTags(service, operation string, header nexus.Header) []metrics.Tag { + conf := h.metricTagConfig() + var tags []metrics.Tag + if conf.IncludeServiceTag { + tags = append(tags, metrics.NexusServiceTag(service)) + } + if conf.IncludeOperationTag { + tags = append(tags, metrics.NexusOperationTag(operation)) + } + for _, mapping := range conf.HeaderTagMappings { + tags = append(tags, metrics.StringTag(mapping.TargetTag, header.Get(mapping.SourceHeader))) + } + return tags } // Extracts a nexusContext from the given ctx and returns an operationContext with tagged metrics and logging. @@ -255,24 +300,20 @@ func (h *nexusHandler) getOperationContext(ctx context.Context, method string) ( clientVersionChecker: headers.NewDefaultVersionChecker(), requestErrorHandler: h.requestErrorHandler, headersBlacklist: h.headersBlacklist, - metricTagConfig: h.metricTagConfig, } oc.metricsHandlerForInterceptors = h.metricsHandler.WithTags( metrics.OperationTag(method), metrics.NamespaceTag(nc.namespaceName), ) - oc.metricsHandler = h.metricsHandler.WithTags( - metrics.NamespaceTag(nc.namespaceName), - metrics.NexusEndpointTag(nc.endpointName), - metrics.NexusMethodTag(method), - // default to internal error unless overridden by handler - metrics.OutcomeTag("internal_error"), - ) var err error if oc.namespace, err = h.namespaceRegistry.GetNamespace(namespace.Name(nc.namespaceName)); err != nil { - metrics.NexusRequests.With(oc.metricsHandler).Record( + // draft-review: should this block be removed now that this is in an interceptor? + metrics.NexusRequests.With(h.metricsHandler).Record( 1, + metrics.NamespaceTag(nc.namespaceName), + metrics.NexusEndpointTag(nc.endpointName), + metrics.NexusMethodTag(method), metrics.OutcomeTag("namespace_not_found"), ) @@ -297,10 +338,61 @@ func (h *nexusHandler) StartOperation( return nil, err } ctx = oc.augmentContext(ctx, options.Header) - oc.enrichNexusOperationMetrics(service, operation, options.Header) oc.enrichNexusOperationLogs(service, operation, options.RequestID) - defer captureOperationPanic(oc.logger, &retErr) + oc.annotateServerSpan(ctx, service, operation, options.RequestID) + // to handle edge case where the operation panics before the interceptor chain is invoked + defer finalizeOperationRequest(oc, &retErr) + + ctx = withOperationContext(ctx, oc) + + nexusOpInput := interceptornexus.NewStartOpInput( + service, + operation, + oc.namespaceName, + options, + input, + interceptornexus.ForwardingInfo{ + OriginalRequestHeaders: oc.originalRequestHeaders, + TaskQueue: oc.taskQueue, + EndpointID: oc.endpointID, + EndpointName: oc.endpointName, + }, + interceptornexus.RequestMetadata{ + APIName: oc.apiName, + NamespaceEntry: oc.namespace, + EndpointName: oc.endpointName, + MetricTags: h.nexusMetricTags(service, operation, options.Header), + }, + ) + out, err := h.chainedHandler(ctx, nexusOpInput) + if err != nil { + return nil, err + } + res, ok := out.(nexus.HandlerStartOperationResult[any]) + if !ok { + return nil, fmt.Errorf("unexpected Nexus start interceptor result type %T", out) + } + return res, nil +} +//nolint:revive,cognitive-complexity: justified to keep the flow intact +func (h *nexusHandler) finalStartHandler( + ctx context.Context, + in interceptornexus.InterceptorInput, +) (any, error) { + oc, ocok := operationContextFromContext(ctx) + if !ocok { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid operation context for nexus start operation") + } + operation := in.OperationName() + var input *nexus.LazyValue + var options nexus.StartOperationOptions + if soi, ok := in.(interceptornexus.StartOpInput); !ok { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid request for nexus start operation") + } else { + input = soi.StartOperationInput + options = soi.StartOperationOptions + } var links []*nexuspb.Link for _, nexusLink := range options.Links { links = append(links, &nexuspb.Link{ @@ -308,9 +400,8 @@ func (h *nexusHandler) StartOperation( Type: nexusLink.Type, }) } - - startOperationRequest := nexuspb.StartOperationRequest{ - Service: service, + startOperationRequest := &nexuspb.StartOperationRequest{ + Service: in.ServiceName(), Operation: operation, Callback: options.CallbackURL, CallbackHeader: options.CallbackHeader, @@ -321,53 +412,13 @@ func (h *nexusHandler) StartOperation( ScheduledTime: timestamppb.New(oc.requestStartTime), Header: options.Header, Variant: &nexuspb.Request_StartOperation{ - StartOperation: &startOperationRequest, + StartOperation: startOperationRequest, }, Capabilities: &nexuspb.Request_Capabilities{ TemporalFailureResponses: oc.callerFailureSupport, }, }) - - finalHandler := func(ctx context.Context, _ interceptornexus.InterceptorInput) (any, error) { - return h.finalStartHandler(ctx, oc, operation, input, &startOperationRequest, request) - } - - nexusOpInput := interceptornexus.NewStartOpInput(service, operation, oc.namespaceName, options, input) - nexusOpInput.WithForwardingInfo(interceptornexus.ForwardingInfo{ - OriginalRequestHeaders: oc.originalRequestHeaders, - TaskQueue: oc.taskQueue, - EndpointID: oc.endpointID, - EndpointName: oc.endpointName, - }) - nexusOpInput.WithRequestMetadata(interceptornexus.RequestMetadata{ - APIName: oc.apiName, - NamespaceEntry: oc.namespace, - EndpointName: oc.endpointName, - }) - chainedHandler := interceptornexus.ChainInterceptors(finalHandler, h.nexusInterceptors) - out, err := chainedHandler(ctx, nexusOpInput) - if err != nil { - if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { - return nil, taggedErr.Err - } - return nil, err - } - res, ok := out.(nexus.HandlerStartOperationResult[any]) - if !ok { - return nil, fmt.Errorf("unexpected Nexus start interceptor result type %T", out) - } - return res, nil -} - -//nolint:revive,cognitive-complexity: this is just a shift of existing code -func (h *nexusHandler) finalStartHandler(ctx context.Context, - oc *operationContext, - operation string, - input *nexus.LazyValue, - startOperationRequest *nexuspb.StartOperationRequest, - request *matchingservice.DispatchNexusTaskRequest, -) (any, error) { - oc.sanitizeRequestHeaders(request) + h.sanitizeRequestHeaders(request) var err error // Transform nexus Content to temporal Payload with common/nexus PayloadSerializer. if err = input.Consume(&startOperationRequest.Payload); err != nil { @@ -384,9 +435,11 @@ func (h *nexusHandler) finalStartHandler(ctx context.Context, // RPC. response, err := h.matchingClient.DispatchNexusTask(ctx, request) if err != nil { - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("matching_timeout")) oc.logger.Error("received error from matching service for Nexus StartOperation request", tag.Error(err)) - return nil, commonnexus.ConvertGRPCError(err, false) + return nil, &interceptornexus.InterceptorError{ + Err: err, + Outcome: "matching_timeout", + } } // Convert to standard Nexus SDK response. result, handlerLinks, err := oc.handleStartOperationResponse(response, operation) @@ -421,19 +474,74 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, return err } ctx = oc.augmentContext(ctx, options.Header) - oc.enrichNexusOperationMetrics(service, operation, options.Header) oc.enrichNexusOperationLogs(service, operation, "") - defer captureOperationPanic(oc.logger, &retErr) + oc.annotateServerSpan(ctx, service, operation, "") + // for edge case where the operation panics before the interceptor chain is invoked + defer finalizeOperationRequest(oc, &retErr) + + nexusInterceptorInput := interceptornexus.NewCancelOpInput( + service, + operation, + oc.namespaceName, + options, + token, + interceptornexus.ForwardingInfo{ + OriginalRequestHeaders: oc.originalRequestHeaders, + TaskQueue: oc.taskQueue, + EndpointID: oc.endpointID, + EndpointName: oc.endpointName, + }, + interceptornexus.RequestMetadata{ + APIName: oc.apiName, + NamespaceEntry: oc.namespace, + EndpointName: oc.endpointName, + MetricTags: h.nexusMetricTags(service, operation, options.Header), + }, + ) + ctx = withOperationContext(ctx, oc) + _, err = h.chainedHandler(ctx, nexusInterceptorInput) + return err +} + +func (h *nexusHandler) finalHandler( + ctx context.Context, + in interceptornexus.InterceptorInput, +) (any, error) { + switch in.(type) { + case interceptornexus.StartOpInput: + return h.finalStartHandler(ctx, in) + case interceptornexus.CancelOpInput: + return h.finalCancelHandler(ctx, in) + default: + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "unknown operation triggered, expected start/cancel nexus op") + } +} +func (h *nexusHandler) finalCancelHandler( + ctx context.Context, + in interceptornexus.InterceptorInput, +) (any, error) { + oc, ocok := operationContextFromContext(ctx) + if !ocok { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid operation context for nexus cancel operation") + } + coi, ok := in.(interceptornexus.CancelOpInput) + if !ok { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid request for nexus cancel operation") + } + options := coi.CancelOperationOptions + token := coi.CancellationToken + + operation := in.OperationName() request := oc.matchingRequest(&nexuspb.Request{ Header: options.Header, ScheduledTime: timestamppb.New(oc.requestStartTime), Variant: &nexuspb.Request_CancelOperation{ CancelOperation: &nexuspb.CancelOperationRequest{ - Service: service, + Service: in.ServiceName(), Operation: operation, OperationToken: token, - // TODO(bergundy): Remove this fallback after the 1.27 release. + // TODO(bergundy): Remove this fallback after the 1.27 release. - can this be removed now? OperationId: token, }, }, @@ -441,147 +549,21 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, TemporalFailureResponses: oc.callerFailureSupport, }, }) - - finalHandler := func(ctx context.Context, _ interceptornexus.InterceptorInput) (any, error) { - return nil, h.finalCancelHandler(ctx, oc, operation, request) - } - - nexusInterceptorInput := interceptornexus.NewCancelOpInput(service, operation, oc.namespaceName, options, token) - nexusInterceptorInput.WithForwardingInfo(interceptornexus.ForwardingInfo{ - OriginalRequestHeaders: oc.originalRequestHeaders, - TaskQueue: oc.taskQueue, - EndpointID: oc.endpointID, - EndpointName: oc.endpointName, - }) - nexusInterceptorInput.WithRequestMetadata(interceptornexus.RequestMetadata{ - APIName: oc.apiName, - NamespaceEntry: oc.namespace, - EndpointName: oc.endpointName, - }) - chainedHandler := interceptornexus.ChainInterceptors(finalHandler, h.nexusInterceptors) - _, err = chainedHandler(ctx, nexusInterceptorInput) - if err != nil { - if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { - return taggedErr.Err - } - return err - } - return nil -} - -func (h *nexusHandler) finalCancelHandler( - ctx context.Context, - oc *operationContext, - operation string, - request *matchingservice.DispatchNexusTaskRequest, -) error { - oc.sanitizeRequestHeaders(request) + h.sanitizeRequestHeaders(request) // Dispatch the request to be sync matched with a worker polling on the nexusContext taskQueue. // matchingClient sets a context timeout of 60 seconds for this request, this should be enough for any Nexus // RPC. response, err := h.matchingClient.DispatchNexusTask(ctx, request) if err != nil { - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("matching_timeout")) oc.logger.Error("received error from matching service for Nexus CancelOperation request", tag.Error(err)) - return commonnexus.ConvertGRPCError(err, false) - } - // Convert to standard Nexus SDK response. - return oc.handleCancelOperationResponse(response, operation) -} - -func (h *nexusHandler) forwardCancelOperation( - ctx context.Context, - service string, - operation string, - id string, - options nexus.CancelOperationOptions, - oc *operationContext, -) error { - options.Header[interceptor.DCRedirectionAPIHeaderName] = "true" - options.Header[interceptor.DCRedirectionSourceCellHeaderName] = h.clusterMetadata.GetCurrentClusterName() - - client, err := h.nexusClientForActiveCluster(oc, service) - if err != nil { - return err - } - - handle, err := client.NewOperationHandle(operation, id) - if err != nil { - oc.logger.Warn("invalid Nexus cancel operation.", tag.Error(err)) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation") - } - - if h.httpTraceProvider != nil { - traceLogger := log.With(h.logger, - tag.Operation(oc.method), - tag.WorkflowNamespace(oc.namespaceName), - tag.NexusOperation(operation), - tag.Endpoint(oc.endpointName), - tag.AttemptStart(time.Now().UTC()), - tag.SourceCluster(h.clusterMetadata.GetCurrentClusterName()), - tag.TargetCluster(oc.namespace.ActiveClusterName(namespace.RoutingKey{})), - ) - if trace := h.httpTraceProvider.NewForwardingTrace(traceLogger); trace != nil { - ctx = httptrace.WithClientTrace(ctx, trace) + return nil, &interceptornexus.InterceptorError{ + Err: err, + Outcome: "matching_timeout", } } - - err = handle.Cancel(ctx, options) - if err != nil { - oc.logger.Error("received error from remote cluster for forwarded Nexus cancel operation request.", tag.Error(err)) - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("forwarded_request_error")) - return err - } - - return nil -} - -func (h *nexusHandler) nexusClientForActiveCluster(oc *operationContext, service string) (*nexusrpc.HTTPClient, error) { - httpClient, err := h.forwardingClients.Get(oc.namespace.ActiveClusterName(namespace.RoutingKey{})) - if err != nil { - oc.logger.Error("failed to forward Nexus request. error creating HTTP client", tag.Error(err), tag.SourceCluster(oc.namespace.ActiveClusterName(namespace.RoutingKey{})), tag.TargetCluster(oc.namespace.ActiveClusterName(namespace.RoutingKey{}))) - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("request_forwarding_failed")) - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed") - } - - httpCaller := &forwardingHttpHeaderWrapper{ - client: httpClient, - nc: oc.nexusContext, - originalRequestHeaders: oc.originalRequestHeaders, - } - - var baseURL string - if h.useForwardByEndpoint() && oc.endpointID != "" { - // If the request was originally dispatched by endpoint, forward by endpoint as well. - baseURL, err = url.JoinPath(httpClient.BaseURL(), - commonnexus.RouteDispatchNexusTaskByEndpoint.Path(oc.endpointID)) - } else { - // Fallback to dispatch by namespace and task queue since those have already been resolved by this point. - // NOTE: When forwarding by namespace and task queue, the endpoint name is not preserved and cannot be provided to a worker polling. - baseURL, err = url.JoinPath( - httpClient.BaseURL(), - commonnexus.RouteDispatchNexusTaskByNamespaceAndTaskQueue.Path(commonnexus.NamespaceAndTaskQueue{ - Namespace: oc.namespaceName, - TaskQueue: oc.taskQueue, - })) - } - - if err != nil { - oc.logger.Error("failed to forward Nexus request. error constructing ServiceBaseURL", - tag.URL(httpClient.BaseURL()), - tag.WorkflowNamespace(oc.namespaceName), - tag.WorkflowTaskQueueName(oc.taskQueue), - tag.Error(err)) - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("request_forwarding_failed")) - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed") - } - - return nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ - HTTPCaller: httpCaller.Do, - BaseURL: baseURL, - Service: service, - }) + // Convert to standard Nexus SDK response. + return nil, oc.handleCancelOperationResponse(response, operation) } func convertOutcomeToNexusHandlerError(resp *matchingservice.DispatchNexusTaskResponse_HandlerError) *nexus.HandlerError { diff --git a/service/frontend/nexus_handler_test.go b/service/frontend/nexus_handler_test.go index b553222e3bf..81138006b53 100644 --- a/service/frontend/nexus_handler_test.go +++ b/service/frontend/nexus_handler_test.go @@ -14,12 +14,10 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" - "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/quotas" - "go.temporal.io/server/common/rpc/interceptor" ) type mockAuthorizer struct{} @@ -76,9 +74,7 @@ func newOperationContext(options contextOptions) *operationContext { nexusContext: &nexusContext{}, } oc.logger = log.NewTestLogger() - mh := metricstest.NewCaptureHandler() - oc.metricsHandlerForInterceptors = mh - oc.metricsHandler = mh + oc.metricsHandlerForInterceptors = metricstest.NewCaptureHandler() oc.clientVersionChecker = headers.NewDefaultVersionChecker() oc.apiName = "/temporal.api.nexusservice.v1.NexusService/DispatchNexusTask" oc.responseHeaders = make(map[string]string) @@ -108,28 +104,6 @@ func newOperationContext(options contextOptions) *operationContext { 1, ) - oc.namespaceConcurrencyLimitInterceptor = interceptor.NewConcurrentRequestLimitInterceptor( - nil, - nil, - oc.logger, - func(ns string) int { return options.quota }, - func(ns string) int { return options.quota }, - map[string]int{ - oc.apiName: 1, - }, - ) - oc.namespaceRateLimitInterceptor = interceptor.NewNamespaceRateLimitInterceptor( - nil, - mockRateLimiter{options.namespaceRateLimitAllow}, - map[string]struct{}{}, - dynamicconfig.GetBoolPropertyFnFilteredByNamespace(false), - metrics.NoopMetricsHandler, - ) - oc.rateLimitInterceptor = interceptor.NewRateLimitInterceptor( - mockRateLimiter{options.rateLimitAllow}, - make(map[string]int), - ) - oc.clusterMetadata = clustertest.NewMetadataForTest( cluster.NewTestClusterMetadataConfig(true, !options.namespacePassive), ) diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index cecd8cdb686..ccb0f3bbe62 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -27,7 +27,6 @@ import ( "go.temporal.io/server/common/routing" "go.temporal.io/server/common/rpc" "go.temporal.io/server/common/rpc/interceptor" - interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/telemetry" "go.temporal.io/server/service/frontend/configs" "google.golang.org/grpc/codes" @@ -37,18 +36,15 @@ import ( // Small wrapper that does some pre-processing before handing requests over to the Nexus SDK's HTTP handler. type NexusOperationHTTPHandler struct { - base nexusrpc.BaseHTTPHandler - logger log.Logger - nexusHandler http.Handler - enpointRegistry commonnexus.EndpointRegistry - namespaceRegistry namespace.Registry - preprocessErrorCounter metrics.CounterFunc - auth *authorization.Interceptor - namespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor - namespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor - namespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor - rateLimitInterceptor *interceptor.RateLimitInterceptor - httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter + base nexusrpc.BaseHTTPHandler + logger log.Logger + nexusHandler http.Handler + enpointRegistry commonnexus.EndpointRegistry + namespaceRegistry namespace.Registry + namespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor + preprocessErrorCounter metrics.CounterFunc + auth *authorization.Interceptor + httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter } func NewNexusOperationHTTPHandler( @@ -60,19 +56,9 @@ func NewNexusOperationHTTPHandler( namespaceRegistry namespace.Registry, endpointRegistry commonnexus.EndpointRegistry, authInterceptor *authorization.Interceptor, - telemetryInterceptor *interceptor.TelemetryInterceptor, - requestErrorHandler *interceptor.RequestErrorHandler, - redirectionInterceptor *interceptor.Redirection, namespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor, - namespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor, - nexusNamespaceRateLimitInterceptor *interceptor.NamespaceRateLimitInterceptorWrapper, - namespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor, - rateLimitInterceptor *interceptor.RateLimitInterceptor, - sdkVersionInterceptor *interceptor.SDKVersionInterceptor, - callerInfoInterceptor *interceptor.CallerInfoInterceptor, - nexusForwarder *nexusForwardingInterceptor, + requestErrorHandler *interceptor.RequestErrorHandler, interceptorsProvider *InterceptorsProvider, - customNexusInterceptors []interceptornexus.Interceptor, logger log.Logger, httpTraceProvider commonnexus.HTTPClientTraceProvider, httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter, @@ -84,31 +70,28 @@ func NewNexusOperationHTTPHandler( Logger: log.NewSlogLogger(logger), FailureConverter: nexusrpc.DefaultFailureConverter(), }, - logger: logger, - enpointRegistry: endpointRegistry, - namespaceRegistry: namespaceRegistry, - auth: authInterceptor, - namespaceValidationInterceptor: namespaceValidationInterceptor, - namespaceRateLimitInterceptor: namespaceRateLimitInterceptor, - namespaceConcurrencyLimitInterceptor: namespaceConcurrencyLimitInterceptor, - rateLimitInterceptor: rateLimitInterceptor, - preprocessErrorCounter: metricsHandler.Counter(metrics.NexusRequestPreProcessErrors.Name()).Record, - httpServerHandlerInstrumenter: httpServerHandlerInstrumenter, + logger: logger, + enpointRegistry: endpointRegistry, + namespaceRegistry: namespaceRegistry, + auth: authInterceptor, + namespaceValidationInterceptor: namespaceValidationInterceptor, + preprocessErrorCounter: metricsHandler.Counter(metrics.NexusRequestPreProcessErrors.Name()).Record, + httpServerHandlerInstrumenter: httpServerHandlerInstrumenter, nexusHandler: nexusrpc.NewHTTPHandler(nexusrpc.HandlerOptions{ - Handler: &nexusHandler{ - logger: logger, - metricsHandler: metricsHandler, - clusterMetadata: clusterMetadata, - namespaceRegistry: namespaceRegistry, - matchingClient: matchingservice.MatchingServiceClient(matchingClient), - requestErrorHandler: requestErrorHandler, - payloadSizeLimit: serviceConfig.BlobSizeLimitError, - headersBlacklist: serviceConfig.NexusRequestHeadersBlacklist, - useForwardByEndpoint: serviceConfig.NexusForwardRequestUseEndpoint, - metricTagConfig: serviceConfig.NexusOperationsMetricTagConfig, - httpTraceProvider: httpTraceProvider, - nexusInterceptors: interceptorsProvider.GetNexusInterceptors(), - }, + Handler: newNexusHandler( + logger, + metricsHandler, + clusterMetadata, + namespaceRegistry, + matchingservice.MatchingServiceClient(matchingClient), + requestErrorHandler, + serviceConfig.BlobSizeLimitError, + serviceConfig.NexusRequestHeadersBlacklist, + serviceConfig.NexusForwardRequestUseEndpoint, + serviceConfig.NexusOperationsMetricTagConfig, + httpTraceProvider, + interceptorsProvider.NexusInterceptors(), + ), GetResultTimeout: serviceConfig.KeepAliveMaxConnectionIdle(), Logger: log.NewSlogLogger(logger), Serializer: commonnexus.PayloadSerializer, @@ -250,14 +233,10 @@ func (h *NexusOperationHTTPHandler) dispatchNexusTaskByEndpoint(w http.ResponseW func (h *NexusOperationHTTPHandler) baseNexusContext(apiName string, header http.Header) *nexusContext { return &nexusContext{ - namespaceValidationInterceptor: h.namespaceValidationInterceptor, - namespaceRateLimitInterceptor: h.namespaceRateLimitInterceptor, - namespaceConcurrencyLimitInterceptor: h.namespaceConcurrencyLimitInterceptor, - rateLimitInterceptor: h.rateLimitInterceptor, - apiName: apiName, - requestStartTime: time.Now(), - responseHeaders: make(map[string]string), - callerFailureSupport: header.Get(nexusrpc.HeaderTemporalNexusFailureSupport) == "true", + apiName: apiName, + requestStartTime: time.Now(), + responseHeaders: make(map[string]string), + callerFailureSupport: header.Get(nexusrpc.HeaderTemporalNexusFailureSupport) == "true", } } @@ -327,14 +306,13 @@ func (h *NexusOperationHTTPHandler) parseTLSAndAuthInfo(r *http.Request, nc *nex return "" // TODO: support audience getter }) - var err error if authInfo != nil { - nc.claims, err = h.auth.GetClaims(authInfo) + claims, err := h.auth.GetClaims(authInfo) if err != nil { return nil, err } // Make the auth info and claims available on the context. - r = r.WithContext(h.auth.EnhanceContext(r.Context(), authInfo, nc.claims)) + r = r.WithContext(h.auth.EnhanceContext(r.Context(), authInfo, claims)) } return r, nil diff --git a/temporal/fx.go b/temporal/fx.go index dd70640dc23..398aecd7440 100644 --- a/temporal/fx.go +++ b/temporal/fx.go @@ -47,7 +47,6 @@ import ( "go.temporal.io/server/common/resource" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" - "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" "go.temporal.io/server/common/telemetry" @@ -122,7 +121,7 @@ type ( TokenProvider auth.TokenProvider ServiceHosts map[primitives.ServiceName]static.Hosts - CustomFrontendNexusInterceptors []nexus.Interceptor + CustomFrontendUnifiedInterceptors []frontend.Interceptor // below are things that could be over write by server options or may have default if not supplied by serverOptions. Logger log.Logger @@ -321,12 +320,12 @@ func ServerOptionsProvider(opts []ServerOption) (serverOptionsProvider, error) { ServiceHosts: so.hostsByService, NamespaceLogger: so.namespaceLogger, - ServiceResolver: so.persistenceServiceResolver, - CustomDataStoreFactory: so.customDataStoreFactory, - CustomVisibilityStore: so.customVisibilityStoreFactory, - CustomHistoryArchiverFactory: so.customHistoryArchiverFactory, - CustomVisibilityArchiverFactory: so.customVisibilityArchiverFactory, - CustomFrontendNexusInterceptors: so.customFrontendUnifiedInterceptors, + ServiceResolver: so.persistenceServiceResolver, + CustomDataStoreFactory: so.customDataStoreFactory, + CustomVisibilityStore: so.customVisibilityStoreFactory, + CustomHistoryArchiverFactory: so.customHistoryArchiverFactory, + CustomVisibilityArchiverFactory: so.customVisibilityArchiverFactory, + CustomFrontendUnifiedInterceptors: so.customFrontendUnifiedInterceptors, SearchAttributesMapper: so.searchAttributesMapper, CustomFrontendInterceptors: so.customFrontendInterceptors, @@ -384,37 +383,37 @@ type ( ServiceProviderParamsCommon struct { fx.In - Cfg *config.Config - ServiceNames resource.ServiceNames - Logger log.Logger - NamespaceLogger resource.NamespaceLogger - DynamicConfigClient dynamicconfig.Client - MetricsHandler metrics.Handler - EventLoggerProvider otellog.LoggerProvider - EsClient esclient.Client - TlsConfigProvider encryption.TLSConfigProvider //nolint:staticcheck // should be TLSConfigProvider - PersistenceConfig config.Persistence - ClusterMetadata *cluster.Config - ClientFactoryProvider client.FactoryProvider - AudienceGetter authorization.JWTAudienceMapper - PersistenceServiceResolver resolver.ServiceResolver - PersistenceFactoryProvider persistenceClient.FactoryProviderFn - SearchAttributesMapper searchattribute.Mapper - CustomFrontendInterceptors []grpc.UnaryServerInterceptor - CustomFrontendNexusInterceptors []nexus.Interceptor - AdditionalStreamInterceptors []grpc.StreamServerInterceptor - Authorizer authorization.Authorizer - ClaimMapper authorization.ClaimMapper - TokenProvider auth.TokenProvider - DataStoreFactory persistenceClient.AbstractDataStoreFactory - VisibilityStoreFactory visibility.VisibilityStoreFactory - CustomHistoryArchiverFactory provider.CustomHistoryArchiverFactory - CustomVisibilityArchiverFactory provider.CustomVisibilityArchiverFactory - SpanExporters []otelsdktrace.SpanExporter - InstanceID resource.InstanceID `optional:"true"` - StaticServiceHosts map[primitives.ServiceName]static.Hosts `optional:"true"` - TaskCategoryRegistry tasks.TaskCategoryRegistry - TestHooks testhooks.TestHooks + Cfg *config.Config + ServiceNames resource.ServiceNames + Logger log.Logger + NamespaceLogger resource.NamespaceLogger + DynamicConfigClient dynamicconfig.Client + MetricsHandler metrics.Handler + EventLoggerProvider otellog.LoggerProvider + EsClient esclient.Client + TlsConfigProvider encryption.TLSConfigProvider //nolint:staticcheck // should be TLSConfigProvider + PersistenceConfig config.Persistence + ClusterMetadata *cluster.Config + ClientFactoryProvider client.FactoryProvider + AudienceGetter authorization.JWTAudienceMapper + PersistenceServiceResolver resolver.ServiceResolver + PersistenceFactoryProvider persistenceClient.FactoryProviderFn + SearchAttributesMapper searchattribute.Mapper + CustomFrontendInterceptors []grpc.UnaryServerInterceptor + CustomFrontendUnifiedInterceptors []frontend.Interceptor + AdditionalStreamInterceptors []grpc.StreamServerInterceptor + Authorizer authorization.Authorizer + ClaimMapper authorization.ClaimMapper + TokenProvider auth.TokenProvider + DataStoreFactory persistenceClient.AbstractDataStoreFactory + VisibilityStoreFactory visibility.VisibilityStoreFactory + CustomHistoryArchiverFactory provider.CustomHistoryArchiverFactory + CustomVisibilityArchiverFactory provider.CustomVisibilityArchiverFactory + SpanExporters []otelsdktrace.SpanExporter + InstanceID resource.InstanceID `optional:"true"` + StaticServiceHosts map[primitives.ServiceName]static.Hosts `optional:"true"` + TaskCategoryRegistry tasks.TaskCategoryRegistry + TestHooks testhooks.TestHooks } ) @@ -599,7 +598,7 @@ func genericFrontendServiceProvider( app := fx.New( params.GetCommonServiceOptions(serviceName), fx.Supply(params.CustomFrontendInterceptors), - fx.Supply(params.CustomFrontendNexusInterceptors), + fx.Supply(params.CustomFrontendUnifiedInterceptors), fx.Decorate(func() authorization.ClaimMapper { switch serviceName { case primitives.FrontendService: diff --git a/temporal/server_option.go b/temporal/server_option.go index a86e5daa151..04498b9e61c 100644 --- a/temporal/server_option.go +++ b/temporal/server_option.go @@ -18,9 +18,9 @@ import ( "go.temporal.io/server/common/resolver" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" - "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/service/frontend" "google.golang.org/grpc" ) @@ -210,11 +210,12 @@ func WithChainedFrontendGrpcInterceptors( }) } -// TBD: this will become unified interceptors instead -// -//nolint:staticcheck -func WithChainedFrontendNexusInterceptors( - interceptors ...nexus.Interceptor, +// WithChainedFrontendInterceptors sets an ordered chain of custom gRPC+Nexus interceptors that will be invoked for all +// Frontend gRPC and Nexus API calls respectively. The list of custom interceptors will be appended to the end of the internal +// ServerInterceptors. The custom interceptors will be invoked in the order as they appear in the supplied list, after +// the internal ServerInterceptors. +func WithChainedFrontendInterceptors( + interceptors ...frontend.Interceptor, ) ServerOption { return applyFunc(func(s *serverOptions) { s.customFrontendUnifiedInterceptors = interceptors diff --git a/temporal/server_options.go b/temporal/server_options.go index b161d241e91..123a0c19e6d 100644 --- a/temporal/server_options.go +++ b/temporal/server_options.go @@ -21,9 +21,9 @@ import ( "go.temporal.io/server/common/resolver" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" - "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/service/frontend" "google.golang.org/grpc" ) @@ -62,7 +62,7 @@ type ( persistenceFactoryProvider persistenceClient.FactoryProviderFn searchAttributesMapper searchattribute.Mapper customFrontendInterceptors []grpc.UnaryServerInterceptor - customFrontendUnifiedInterceptors []nexus.Interceptor + customFrontendUnifiedInterceptors []frontend.Interceptor additionalStreamInterceptors []grpc.StreamServerInterceptor metricHandler metrics.Handler eventLoggerProvider otellog.LoggerProvider diff --git a/tests/nexus_api_validation_test.go b/tests/nexus_api_validation_test.go index 354145a2994..dde819faddf 100644 --- a/tests/nexus_api_validation_test.go +++ b/tests/nexus_api_validation_test.go @@ -201,7 +201,7 @@ func (s *NexusAPIValidationTestSuite) TestNexusStartOperation_Forbidden() { client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{BaseURL: dispatchURL, Service: "test-service"}) s.NoError(err) - // capture := env.StartNamespaceMetricCapture() + capture := env.StartNamespaceMetricCapture() _, err = nexusrpc.StartOperation(s.Context(), client, op, "input", nexus.StartOperationOptions{}) @@ -209,10 +209,10 @@ func (s *NexusAPIValidationTestSuite) TestNexusStartOperation_Forbidden() { s.ErrorAs(err, &handlerErr) tc.checkFailure(s, handlerErr) - // requests := capture.Metric("nexus_requests") - // s.Len(requests, 1) - // s.Subset(requests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "method": "StartNexusOperation", "outcome": tc.expectedOutcomeMetric}) - // s.Equal(int64(1), requests[0].Value) + requests := capture.Metric("nexus_requests") + s.Len(requests, 1) + s.Subset(requests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "method": "StartNexusOperation", "outcome": tc.expectedOutcomeMetric}) + s.Equal(int64(1), requests[0].Value) } for _, tc := range testCases { diff --git a/tests/nexus_workflow_test.go b/tests/nexus_workflow_test.go index 8adb7985cdb..abef91f61e1 100644 --- a/tests/nexus_workflow_test.go +++ b/tests/nexus_workflow_test.go @@ -1782,14 +1782,14 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationAsyncCompletionAuthErrors(cha } publicCallbackURL := "http://" + env.HttpAPIAddress() + "/" + commonnexus.RouteCompletionCallback.Path(env.Namespace().String()) - // capture := env.StartNamespaceMetricCapture() + capture := env.StartNamespaceMetricCapture() err = s.sendNexusCompletionRequest(s.Context(), publicCallbackURL, completion) - // completionRequests := capture.Metric("nexus_completion_requests") + completionRequests := capture.Metric("nexus_completion_requests") var handlerErr *nexus.HandlerError s.ErrorAs(err, &handlerErr) s.Equal(nexus.HandlerErrorTypeUnauthorized, handlerErr.Type) - // s.Len(completionRequests, 1) - // s.Subset(completionRequests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "outcome": "unauthorized"}) + s.Len(completionRequests, 1) + s.Subset(completionRequests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "outcome": "unauthorized"}) } func (s *NexusWorkflowTestSuite) TestNexusOperationAsyncCompletionAuthErrorsNoIdentifier(chasmEnabled bool) { @@ -1812,14 +1812,14 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationAsyncCompletionAuthErrorsNoId Header: nexus.Header{commonnexus.CallbackTokenHeader: callbackToken}, } publicCallbackURL := "http://" + env.HttpAPIAddress() + commonnexus.PathCompletionCallbackNoIdentifier - // capture := env.StartNamespaceMetricCapture() + capture := env.StartNamespaceMetricCapture() err = s.sendNexusCompletionRequest(s.Context(), publicCallbackURL, completion) - // completionRequests := capture.Metric("nexus_completion_requests") + completionRequests := capture.Metric("nexus_completion_requests") var handlerErr *nexus.HandlerError s.ErrorAs(err, &handlerErr) s.Equal(nexus.HandlerErrorTypeUnauthorized, handlerErr.Type) - // s.Len(completionRequests, 1) - // s.Subset(completionRequests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "outcome": "unauthorized"}) + s.Len(completionRequests, 1) + s.Subset(completionRequests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "outcome": "unauthorized"}) } func (s *NexusWorkflowTestSuite) TestNexusOperationAsyncCompletionInternalAuth(chasmEnabled bool) { diff --git a/tests/xdc/nexus_request_forwarding_test.go b/tests/xdc/nexus_request_forwarding_test.go index 359fe42bc78..d3544598782 100644 --- a/tests/xdc/nexus_request_forwarding_test.go +++ b/tests/xdc/nexus_request_forwarding_test.go @@ -131,7 +131,7 @@ func (s *NexusRequestForwardingSuite) TestStartOperationForwardedFromStandbyToAc require.NoError(t, retErr) require.Equal(t, "input", result.Successful) requireExpectedMetricsCaptured(t, activeSnap, ns, "StartNexusOperation", "sync_success") - // requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "request_forwarded") + requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "request_forwarded") }, }, { @@ -180,7 +180,7 @@ func (s *NexusRequestForwardingSuite) TestStartOperationForwardedFromStandbyToAc require.NoError(t, json.Unmarshal(appErrDetails.Details, &details)) require.Equal(t, "details", details) requireExpectedMetricsCaptured(t, activeSnap, ns, "StartNexusOperation", "operation_error") - // requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "forwarded_request_error") + requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "forwarded_request_error") }, }, { @@ -203,7 +203,7 @@ func (s *NexusRequestForwardingSuite) TestStartOperationForwardedFromStandbyToAc require.Error(t, handlerErr.Cause) require.Equal(t, "deliberate internal failure", handlerErr.Cause.Error()) requireExpectedMetricsCaptured(t, activeSnap, ns, "StartNexusOperation", "handler_error:INTERNAL") - // requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "forwarded_request_error") + requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "forwarded_request_error") }, }, { @@ -222,7 +222,7 @@ func (s *NexusRequestForwardingSuite) TestStartOperationForwardedFromStandbyToAc require.ErrorAs(t, retErr, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeUnavailable, handlerErr.Type) require.Equal(t, "cluster inactive", handlerErr.Message) - // requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "namespace_inactive_forwarding_disabled") + requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "namespace_inactive_forwarding_disabled") }, }, } @@ -305,7 +305,7 @@ func (s *NexusRequestForwardingSuite) TestCancelOperationForwardedFromStandbyToA assertion: func(t *testing.T, retErr error, activeSnap map[string][]*metricstest.CapturedRecording, passiveSnap map[string][]*metricstest.CapturedRecording) { require.NoError(t, retErr) requireExpectedMetricsCaptured(t, activeSnap, ns, "CancelNexusOperation", "success") - // requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "request_forwarded") + requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "request_forwarded") }, }, { @@ -328,7 +328,7 @@ func (s *NexusRequestForwardingSuite) TestCancelOperationForwardedFromStandbyToA require.Error(t, handlerErr.Cause) require.Equal(t, "deliberate internal failure", handlerErr.Cause.Error()) requireExpectedMetricsCaptured(t, activeSnap, ns, "CancelNexusOperation", "handler_error:INTERNAL") - // requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "forwarded_request_error") + requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "forwarded_request_error") }, }, { @@ -347,7 +347,7 @@ func (s *NexusRequestForwardingSuite) TestCancelOperationForwardedFromStandbyToA require.ErrorAs(t, retErr, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeUnavailable, handlerErr.Type) require.Equal(t, "cluster inactive", handlerErr.Message) - // requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "namespace_inactive_forwarding_disabled") + requireExpectedMetricsCaptured(t, passiveSnap, ns, "CancelNexusOperation", "namespace_inactive_forwarding_disabled") }, }, } @@ -607,18 +607,18 @@ func (s *NexusRequestForwardingSuite) TestOperationCompletionForwardedFromStandb completion.Header.Set(cnexus.CallbackTokenHeader, callbackToken) snap, err := s.sendNexusCompletionRequest(ctx, s.T(), s.clusters[1], publicCallbackUrl, completion) s.NoError(err) - // s.Len(snap["nexus_completion_requests"], 1) - // s.Subset(snap["nexus_completion_requests"][0].Tags, map[string]string{"namespace": ns, "outcome": "request_forwarded"}) - - // // Ensure that CompleteOperation request is tracked as part of normal service telemetry metrics - // s.Condition(func() bool { - // for _, m := range snap["service_requests"] { - // if opTag, ok := m.Tags["operation"]; ok && opTag == "CompleteNexusOperation" { - // return true - // } - // } - // return false - // }) + s.Len(snap["nexus_completion_requests"], 1) + s.Subset(snap["nexus_completion_requests"][0].Tags, map[string]string{"namespace": ns, "outcome": "request_forwarded"}) + + // Ensure that CompleteOperation request is tracked as part of normal service telemetry metrics + s.Condition(func() bool { + for _, m := range snap["service_requests"] { + if opTag, ok := m.Tags["operation"]; ok && opTag == "CompleteNexusOperation" { + return true + } + } + return false + }) // Resend the request and verify we get a not found error since the operation has already completed. snap, err = s.sendNexusCompletionRequest(ctx, s.T(), s.clusters[0], publicCallbackUrl, completion) From f4b241c8b7d4f74f682e51b606d3281a786dc2c6 Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Sun, 6 Sep 2026 17:39:23 -0700 Subject: [PATCH 11/12] address comments, rebase --- common/authorization/interceptor.go | 30 ++- common/authorization/interceptor_test.go | 12 +- common/rpc/grpcfaults/interceptor.go | 31 +++ common/rpc/interceptor/caller_info_test.go | 6 +- .../concurrent_request_limit_test.go | 3 +- .../rpc/interceptor/frontend_service_error.go | 1 + common/rpc/interceptor/mask_internal_error.go | 18 +- .../rpc/interceptor/namespace_rate_limit.go | 6 +- .../interceptor/namespace_rate_limit_test.go | 17 +- common/rpc/interceptor/namespace_validator.go | 21 +- .../interceptor/namespace_validator_test.go | 8 +- common/rpc/interceptor/nexus/nexus.go | 137 ++++++----- common/rpc/interceptor/nexus/nexus_test.go | 24 ++ common/rpc/interceptor/rate_limit.go | 5 +- common/rpc/interceptor/rate_limit_test.go | 18 +- common/rpc/interceptor/sdk_version.go | 6 +- common/rpc/interceptor/sdk_version_test.go | 3 +- .../interceptor/service_error_interceptor.go | 38 ++- common/rpc/interceptor/slow_request_logger.go | 2 +- .../interceptor/slow_request_logger_test.go | 5 +- common/rpc/interceptor/telemetry.go | 7 +- common/rpc/interceptor/telemetry_test.go | 9 +- service/frontend/frontend_interceptors.go | 101 ++++---- service/frontend/fx.go | 28 +-- .../frontend/nexus_completion_http_handler.go | 62 +++-- service/frontend/nexus_dispatch_result.go | 26 ++- .../frontend/nexus_dispatch_result_test.go | 169 ++++++++++---- service/frontend/nexus_forward_interceptor.go | 127 +++++----- .../nexus_forward_interceptor_test.go | 51 ++-- service/frontend/nexus_handler.go | 219 ++++++++---------- service/frontend/nexus_handler_test.go | 78 +------ .../frontend/nexus_interceptor_chain_test.go | 175 ++++++++++++++ .../frontend/nexus_operation_http_handler.go | 10 +- temporal/server_option.go | 7 +- temporal/server_options.go | 6 + 35 files changed, 913 insertions(+), 553 deletions(-) create mode 100644 service/frontend/nexus_interceptor_chain_test.go diff --git a/common/authorization/interceptor.go b/common/authorization/interceptor.go index 83baa28c4dc..5bc0bb1796d 100644 --- a/common/authorization/interceptor.go +++ b/common/authorization/interceptor.go @@ -166,6 +166,7 @@ func (a *Interceptor) InterceptNexus( next nexus.HandlerFunc, ) (any, error) { a.logger.Debug("authorizing request") + ctx = headers.StripPrincipal(ctx) if a.authorizer == nil { return next(ctx, in) } @@ -173,31 +174,36 @@ func (a *Interceptor) InterceptNexus( apiName := in.APIName() endpointName := in.EndpointName() claims, _ := ctx.Value(MappedClaims).(*Claims) //nolint:revive // unchecked-type-assertion: empty claims will 403 - // draft-review: check if this might be required to preserve compatibility for custom authorizers - // or if its ok since an interface was not already used instead - // switch in.(type) { - // case nexus.StartOpInput, nexus.CancelOpInput: - // case *nexus.CancelOpInput: - // } ct := &CallTarget{ APIName: apiName, NexusEndpointName: endpointName, Namespace: namespaceName, - Request: in, + Request: in.Request(), } principal, err := a.Authorize(ctx, claims, ct) if err != nil { if permissionDeniedError, ok := errors.AsType[*serviceerror.PermissionDenied](err); ok { a.logger.Debug("Request unauthorized") return nil, &nexus.InterceptorError{ - Err: commonnexus.AdaptAuthorizeError(permissionDeniedError), - Outcome: "unauthorized", + Err: commonnexus.AdaptAuthorizeError(permissionDeniedError), + Outcome: "unauthorized", + SkipServiceErrorReporting: true, } } - a.logger.Error("Authorization internal error with processing nexus request", tag.Error(err)) + logTags := []tag.Tag{ + tag.Operation(apiName), + tag.WorkflowNamespace(namespaceName), + tag.Endpoint(endpointName), + tag.Error(err), + } + if operationName := in.OperationName(); operationName != "" { + logTags = append(logTags, tag.NexusOperation(operationName)) + } + a.logger.Error("Authorization internal error with processing nexus request", logTags...) return nil, &nexus.InterceptorError{ - Err: err, - Outcome: "internal_auth_error", + Err: err, + Outcome: "internal_auth_error", + SkipServiceErrorReporting: true, } } if a.enablePrincipalPropagation != nil && a.enablePrincipalPropagation(namespaceName) && principal != nil { diff --git a/common/authorization/interceptor_test.go b/common/authorization/interceptor_test.go index 41cffd15d83..05f4499589f 100644 --- a/common/authorization/interceptor_test.go +++ b/common/authorization/interceptor_test.go @@ -8,6 +8,7 @@ import ( "errors" "slices" "testing" + "time" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" @@ -17,6 +18,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/common/api" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" @@ -72,23 +74,26 @@ func TestAuthorizerInterceptorSuite(t *testing.T) { func (s *authorizerInterceptorSuite) TestInterceptNexus() { apiName, endpoint := "NexusAPI", "endpoint" + authorizationRequest := &matchingservice.DispatchNexusTaskRequest{} input := interceptornexus.NewStartOpInput( "s", "o", testNamespace, + time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{ APIName: apiName, EndpointName: endpoint, + Request: authorizationRequest, }, ) expectedTarget := &CallTarget{ APIName: apiName, NexusEndpointName: endpoint, Namespace: testNamespace, - Request: input, + Request: authorizationRequest, } for _, tc := range []struct { name string @@ -108,8 +113,9 @@ func (s *authorizerInterceptorSuite) TestInterceptNexus() { ctx: context.Background(), authorizationResult: &Result{Decision: DecisionDeny}, expectedError: &interceptornexus.InterceptorError{ - Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnauthorized, "permission denied"), - Outcome: "unauthorized", + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnauthorized, "permission denied"), + Outcome: "unauthorized", + SkipServiceErrorReporting: true, }, }, } { diff --git a/common/rpc/grpcfaults/interceptor.go b/common/rpc/grpcfaults/interceptor.go index 1d7828f65d7..c5100a1f8d4 100644 --- a/common/rpc/grpcfaults/interceptor.go +++ b/common/rpc/grpcfaults/interceptor.go @@ -3,6 +3,7 @@ package grpcfaults import ( "context" + "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -43,3 +44,33 @@ func UnaryServerInterceptor(generator Generator) grpc.UnaryServerInterceptor { return resp, err } } + +func NewFaultsInterceptor(generator Generator) *FaultsInterceptor { + return &FaultsInterceptor{ + h: UnaryServerInterceptor(generator), + } +} + +type FaultsInterceptor struct { + h grpc.UnaryServerInterceptor +} + +func (g *FaultsInterceptor) Intercept( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, +) (any, error) { + if g.h == nil { + return handler(ctx, req) + } + return g.h(ctx, req, info, handler) +} + +func (g *FaultsInterceptor) InterceptNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (any, error) { + return next(ctx, in) +} diff --git a/common/rpc/interceptor/caller_info_test.go b/common/rpc/interceptor/caller_info_test.go index 1196c5635d5..25f5901e1ad 100644 --- a/common/rpc/interceptor/caller_info_test.go +++ b/common/rpc/interceptor/caller_info_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "testing" + "time" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" @@ -131,6 +132,7 @@ func (s *callerInfoSuite) TestIntercept_CallerName() { func (s *callerInfoSuite) TestInterceptNexus() { completeInput, err := interceptornexus.NewCompleteOpInput( testNamespace, + time.Now(), &nexusrpc.CompletionRequest{HTTPRequest: &http.Request{}}, nil, interceptornexus.ForwardingInfo{}, @@ -145,12 +147,12 @@ func (s *callerInfoSuite) TestInterceptNexus() { }{ { name: "start", - input: interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), + input: interceptornexus.NewStartOpInput("s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), expectedOrigin: "StartNexusOperation", }, { name: "cancel - preserves background origin", - input: interceptornexus.NewCancelOpInput("s", "o", testNamespace, nexus.CancelOperationOptions{}, "t", interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), + input: interceptornexus.NewCancelOpInput("s", "o", testNamespace, time.Now(), nexus.CancelOperationOptions{}, "t", interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), callerInfo: headers.SystemBackgroundHighCallerInfo, }, { diff --git a/common/rpc/interceptor/concurrent_request_limit_test.go b/common/rpc/interceptor/concurrent_request_limit_test.go index c6df21b8518..a96a986cf30 100644 --- a/common/rpc/interceptor/concurrent_request_limit_test.go +++ b/common/rpc/interceptor/concurrent_request_limit_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "time" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/assert" @@ -153,7 +154,7 @@ func TestConcurrentRequestLimitInterceptor_InterceptNexus(t *testing.T) { map[string]int{"NexusAPI": 1}, ) input := interceptornexus.NewStartOpInput( - "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + "s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusAPI"}, ) diff --git a/common/rpc/interceptor/frontend_service_error.go b/common/rpc/interceptor/frontend_service_error.go index d9dd4229138..ed6ba1ea9a2 100644 --- a/common/rpc/interceptor/frontend_service_error.go +++ b/common/rpc/interceptor/frontend_service_error.go @@ -35,6 +35,7 @@ func NewFrontendServiceErrorInterceptorWrapper(logger log.Logger) *FrontendServi } } +// NewFrontendServiceErrorInterceptor provides the legacy standalone gRPC Interceptor for existing deployments. func NewFrontendServiceErrorInterceptor(logger log.Logger) grpc.UnaryServerInterceptor { t := NewFrontendServiceErrorInterceptorWrapper(logger) return t.Intercept diff --git a/common/rpc/interceptor/mask_internal_error.go b/common/rpc/interceptor/mask_internal_error.go index e8d99402afd..1efb495e33f 100644 --- a/common/rpc/interceptor/mask_internal_error.go +++ b/common/rpc/interceptor/mask_internal_error.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + nexusrpc "github.com/nexus-rpc/sdk-go/nexus" "go.temporal.io/api/serviceerror" "go.temporal.io/server/common" "go.temporal.io/server/common/api" @@ -70,10 +71,10 @@ func (mi *MaskInternalErrorDetailsInterceptor) InterceptNexus( return resp, err } if ie, ok := errors.AsType[*nexus.InterceptorError](err); ok { - ie.Err = mi.maskUnknownOrInternalErrors(in, in.APIName(), ie.Err) + ie.Err = mi.maskNexusError(in, ie.Err) err = ie } else { - err = mi.maskUnknownOrInternalErrors(in, in.APIName(), err) + err = mi.maskNexusError(in, err) } return resp, err } @@ -86,6 +87,19 @@ func (mi *MaskInternalErrorDetailsInterceptor) shouldMaskErrors(req any) bool { return mi.maskInternalError(ns.String()) } +func (mi *MaskInternalErrorDetailsInterceptor) maskNexusError(in nexus.InterceptorInput, err error) error { + if _, ok := errors.AsType[*nexusrpc.HandlerError](err); ok { + return err + } + if _, ok := errors.AsType[*nexusrpc.OperationError](err); ok { + return err + } + if _, ok := common.GetRPCStatus(err); !ok { + return err + } + return mi.maskUnknownOrInternalErrors(in, in.APIName(), err) +} + func (mi *MaskInternalErrorDetailsInterceptor) maskUnknownOrInternalErrors( req any, fullMethodName string, err error, ) error { diff --git a/common/rpc/interceptor/namespace_rate_limit.go b/common/rpc/interceptor/namespace_rate_limit.go index 2c4266d3dbc..d7f29f9f3a9 100644 --- a/common/rpc/interceptor/namespace_rate_limit.go +++ b/common/rpc/interceptor/namespace_rate_limit.go @@ -98,7 +98,6 @@ func NewNamespaceRateLimitInterceptorWrapper(ni NamespaceRateLimitInterceptor) * } // NamespaceRateLimitInterceptorWrapper is a wrapper on namespace rate limiter -// draft-review: should this interim be removed in favor of a lock step impl w/ deps type NamespaceRateLimitInterceptorWrapper struct { ni NamespaceRateLimitInterceptor } @@ -119,8 +118,9 @@ func (n *NamespaceRateLimitInterceptorWrapper) InterceptNexus( ) (out any, retErr error) { if err := n.ni.Allow(ctx, namespace.Name(in.NamespaceName()), in.APIName(), in.Header()); err != nil { return nil, &nexus.InterceptorError{ - Err: err, - Outcome: "namespace_rate_limited", + Err: err, + Outcome: "namespace_rate_limited", + ExposeDetails: true, } } return next(ctx, in) diff --git a/common/rpc/interceptor/namespace_rate_limit_test.go b/common/rpc/interceptor/namespace_rate_limit_test.go index 90a174dc6af..b90d761e377 100644 --- a/common/rpc/interceptor/namespace_rate_limit_test.go +++ b/common/rpc/interceptor/namespace_rate_limit_test.go @@ -36,29 +36,22 @@ type namespaceRateLimitInterceptorSuite struct { func (s *namespaceRateLimitInterceptorSuite) TestInterceptNexus() { for _, tc := range []struct { name string - apiName string input interceptornexus.InterceptorInput - allow *bool + allow bool nextCalled bool expectedOutcome string }{ - {name: "allowed", apiName: "NexusOperation", input: interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: new(true), nextCalled: true}, - {name: "rate limited", apiName: "NexusOperation", input: interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: new(false), expectedOutcome: "namespace_rate_limited"}, + {name: "allowed", input: interceptornexus.NewStartOpInput("s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: true, nextCalled: true}, + {name: "rate limited", input: interceptornexus.NewStartOpInput("s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), expectedOutcome: "namespace_rate_limited"}, } { s.Run(tc.name, func() { ctx := context.Background() - if tc.allow != nil { - s.mockRateLimiter.EXPECT().Allow(gomock.Any(), gomock.Any()).Return(*tc.allow) - } - input := tc.input - if input == nil { - input = interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}) - } + s.mockRateLimiter.EXPECT().Allow(gomock.Any(), gomock.Any()).Return(tc.allow) nextCalled := false wrapper := NewNamespaceRateLimitInterceptorWrapper(s.newImpl(false)) _, err := wrapper.InterceptNexus( ctx, - input, + tc.input, func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil diff --git a/common/rpc/interceptor/namespace_validator.go b/common/rpc/interceptor/namespace_validator.go index 492aac6dff4..cfd1517141d 100644 --- a/common/rpc/interceptor/namespace_validator.go +++ b/common/rpc/interceptor/namespace_validator.go @@ -156,12 +156,17 @@ func (nsvi *NamespaceStateValidatorInterceptor) InterceptNexus( ns, err := in.NamespaceEntry() if err != nil { return nil, &nexus.InterceptorError{ - Err: err, - Outcome: "interceptor_failed", + Err: err, + Outcome: "interceptor_failed", + SkipServiceErrorReporting: true, } } if len(ns.Info().GetName()) > nsvi.maxNamespaceLength() { - return nil, errNamespaceTooLong + return nil, &nexus.InterceptorError{ + Err: errNamespaceTooLong, + Outcome: "interceptor_failed", + SkipServiceErrorReporting: true, + } } return next(ctx, in) @@ -280,14 +285,16 @@ func (ni *NamespaceValidatorInterceptor) InterceptNexus( namespaceEntry, err := in.NamespaceEntry() if err != nil { return nil, &nexus.InterceptorError{ - Err: err, - Outcome: "interceptor_failed", + Err: err, + Outcome: "interceptor_failed", + SkipServiceErrorReporting: true, } } if err := ni.ValidateState(namespaceEntry, in.APIName(), in.ForwardingInfo().BusinessID); err != nil { return nil, &nexus.InterceptorError{ - Err: err, - Outcome: "invalid_namespace_state", + Err: err, + Outcome: "invalid_namespace_state", + SkipServiceErrorReporting: true, } } return next(ctx, in) diff --git a/common/rpc/interceptor/namespace_validator_test.go b/common/rpc/interceptor/namespace_validator_test.go index 82b733215e5..c7d932ea884 100644 --- a/common/rpc/interceptor/namespace_validator_test.go +++ b/common/rpc/interceptor/namespace_validator_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/google/uuid" "github.com/nexus-rpc/sdk-go/nexus" @@ -129,7 +130,7 @@ func (s *namespaceValidatorSuite) TestInterceptNexus() { { name: "resolved namespace", input: interceptornexus.NewStartOpInput( - "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + "s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{ APIName: api.NexusServicePrefix + "DispatchNexusTask", @@ -147,7 +148,7 @@ func (s *namespaceValidatorSuite) TestInterceptNexus() { { name: "invalid namespace state", input: interceptornexus.NewStartOpInput( - "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + "s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{ APIName: api.NexusServicePrefix + "DispatchNexusTask", @@ -165,7 +166,7 @@ func (s *namespaceValidatorSuite) TestInterceptNexus() { { name: "missing namespace", input: interceptornexus.NewStartOpInput( - "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + "s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusAPI"}, ), @@ -186,6 +187,7 @@ func (s *namespaceValidatorSuite) TestInterceptNexus() { var interceptorErr *interceptornexus.InterceptorError s.ErrorAs(err, &interceptorErr) s.Equal(tc.expectedOutcome, interceptorErr.Outcome) + s.True(interceptorErr.SkipServiceErrorReporting) } else { s.NoError(err) } diff --git a/common/rpc/interceptor/nexus/nexus.go b/common/rpc/interceptor/nexus/nexus.go index 3158add2e97..ab229a43bdd 100644 --- a/common/rpc/interceptor/nexus/nexus.go +++ b/common/rpc/interceptor/nexus/nexus.go @@ -3,10 +3,12 @@ package nexus import ( "context" "errors" + "fmt" "net/http" "slices" "strings" "sync" + "time" "github.com/nexus-rpc/sdk-go/nexus" tokenspb "go.temporal.io/server/api/token/v1" @@ -16,19 +18,6 @@ import ( "go.temporal.io/server/common/nexus/nexusrpc" ) -const ( - methodNameStartNexusOp = "StartNexusOperation" - methodNameCancelNexusOp = "CancelNexusOperation" - methodNameCompleteNexusOp = "CompleteNexusOperation" - // metric tags - OutcomeInternalError = "internal_error" - OutcomeRequestForwarded = "request_forwarded" - outcomeSyncSuccess = "sync_success" - outcomeAsyncSuccess = "async_success" - outcomeSuccess = "success" - outcomeErrorInternal = "error_internal" -) - type HandlerFunc func(ctx context.Context, in InterceptorInput) (any, error) type Interceptor func(ctx context.Context, in InterceptorInput, next HandlerFunc) (any, error) @@ -44,6 +33,9 @@ type InterceptorInput interface { MetricTags() []metrics.Tag Header() headers.HeaderGetter MethodName() string + Request() any + Outcome(out any, err error) string + StartTime() time.Time sealNexusOp() } @@ -65,38 +57,22 @@ type ForwardingInfo struct { type InterceptorError struct { // wrapped error Err error - // Outcome tag for metrics reporting, (draft-review: should Outcomes be enum or at least constants instead) + // Outcome tag for metrics reporting Outcome string + // Flag for propagating the error to the caller as-is without conversion + ExposeDetails bool + // SkipServiceErrorReporting prevents reporting the error as a frontend service failure. + SkipServiceErrorReporting bool } func (t *InterceptorError) Error() string { - return t.Err.Error() + return fmt.Sprintf("interceptor error (%s): %v", t.Outcome, t.Err.Error()) } func (t *InterceptorError) Unwrap() error { return t.Err } -// Outcome derives the outcome metric tag value based on the request type and its result -func Outcome(in InterceptorInput, out any, err error) string { - if _, ok := in.(CompleteOpInput); ok { - return completionOutcome(err) - } - if err != nil { - if ie, ok := errors.AsType[*InterceptorError](err); ok && ie.Outcome != "" { - return ie.Outcome - } - return OutcomeInternalError - } - switch out.(type) { - case *nexus.HandlerStartOperationResultSync[any]: - return outcomeSyncSuccess - case *nexus.HandlerStartOperationResultAsync: - return outcomeAsyncSuccess - } - return outcomeSuccess -} - type outcomeOverrideCtxKey struct{} // OutcomeOverride lets an inner interceptor that short-circuits the chain(eg. request forwarder) @@ -137,23 +113,6 @@ func SetOutcomeOverride(ctx context.Context, v string) { override.Set(v) } -func completionOutcome(err error) string { - if err == nil { - return outcomeSuccess - } - if ie, ok := errors.AsType[*InterceptorError](err); ok { - if ie.Outcome != "" { - return ie.Outcome - } - err = ie.Err - } - // retaining behavior - if handlerErr, ok := errors.AsType[*nexus.HandlerError](err); ok { - return "error_" + strings.ToLower(string(handlerErr.Type)) - } - return outcomeErrorInternal -} - // RequestMetadata carries request metadata resolved by the handler (e.g. after a // namespace registry lookup) that is supplied alongside the rest of the params at // InterceptorInput construction time. @@ -162,16 +121,22 @@ type RequestMetadata struct { NamespaceEntry *namespace.Namespace EndpointName string MetricTags []metrics.Tag // handler-resolved frontend dynamic config for the tags to record + Request any // preserves the request shape passed to custom authorizers } // container for ServiceName(), OperationName(), NamespaceName(), ForwardingInfo(), and // the fields in RequestMetadata. type nexusOpBase struct { serviceName, operation, namespaceName, methodName string - header headers.HeaderGetter - // TBD: ForwardingInfo and RequestMetadata could just collapse into nexusOpBase + + header headers.HeaderGetter forwardingInfo ForwardingInfo requestMetadata RequestMetadata + startTime time.Time +} + +func (b nexusOpBase) StartTime() time.Time { + return b.startTime } func (b nexusOpBase) ServiceName() string { @@ -217,6 +182,10 @@ func (b nexusOpBase) MethodName() string { return b.methodName } +func (b nexusOpBase) Request() any { + return b.requestMetadata.Request +} + func (nexusOpBase) sealNexusOp() {} type StartOpInput struct { @@ -229,6 +198,7 @@ func NewStartOpInput( serviceName string, operation string, namespaceName string, + startTime time.Time, options nexus.StartOperationOptions, input *nexus.LazyValue, forwardingInfo ForwardingInfo, @@ -240,9 +210,10 @@ func NewStartOpInput( operation: operation, namespaceName: namespaceName, header: options.Header, - methodName: methodNameStartNexusOp, + methodName: "StartNexusOperation", forwardingInfo: forwardingInfo, requestMetadata: requestMetadata, + startTime: startTime, }, StartOperationOptions: options, StartOperationInput: input, @@ -259,6 +230,7 @@ func NewCancelOpInput( serviceName string, operation string, namespaceName string, + startTime time.Time, options nexus.CancelOperationOptions, cancellationToken string, forwardingInfo ForwardingInfo, @@ -270,9 +242,10 @@ func NewCancelOpInput( operation: operation, namespaceName: namespaceName, header: options.Header, - methodName: methodNameCancelNexusOp, + methodName: "CancelNexusOperation", forwardingInfo: forwardingInfo, requestMetadata: requestMetadata, + startTime: startTime, }, CancelOperationOptions: options, CancellationToken: cancellationToken, @@ -287,6 +260,7 @@ type CompleteOpInput struct { func NewCompleteOpInput( namespaceName string, + startTime time.Time, request *nexusrpc.CompletionRequest, completion *tokenspb.NexusOperationCompletion, forwardingInfo ForwardingInfo, @@ -295,19 +269,68 @@ func NewCompleteOpInput( if request == nil || request.HTTPRequest == nil { return CompleteOpInput{}, errors.New("nexus completion request not found") } + requestMetadata.Request = request return CompleteOpInput{ nexusOpBase: nexusOpBase{ namespaceName: namespaceName, header: request.HTTPRequest.Header, - methodName: methodNameCompleteNexusOp, + methodName: "CompleteNexusOperation", forwardingInfo: forwardingInfo, requestMetadata: requestMetadata, + startTime: startTime, }, CompletionRequest: request, Completion: completion, }, nil } +func (c CompleteOpInput) Outcome(out any, err error) string { + if err == nil { + return "success" + } + if ie, ok := errors.AsType[*InterceptorError](err); ok { + if ie.Outcome != "" { + return ie.Outcome + } + err = ie.Err + } + // retaining behavior + if handlerErr, ok := errors.AsType[*nexus.HandlerError](err); ok { + return "error_" + strings.ToLower(string(handlerErr.Type)) + } + return "error_internal" +} + +func (s StartOpInput) Outcome(out any, err error) string { + if outcome, ok := errorOutcome(err); ok { + return outcome + } + switch out.(type) { + case *nexus.HandlerStartOperationResultSync[any]: + return "sync_success" + case *nexus.HandlerStartOperationResultAsync: + return "async_success" + } + return "internal_error" +} + +func (c CancelOpInput) Outcome(out any, err error) string { + if outcome, ok := errorOutcome(err); ok { + return outcome + } + return "success" +} + +func errorOutcome(err error) (string, bool) { + if err != nil { + if ie, ok := errors.AsType[*InterceptorError](err); ok && ie.Outcome != "" { + return ie.Outcome, true + } + return "internal_error", true + } + return "", false +} + func ChainInterceptors(final HandlerFunc, chain []Interceptor) HandlerFunc { for _, curr := range slices.Backward(chain) { next := final diff --git a/common/rpc/interceptor/nexus/nexus_test.go b/common/rpc/interceptor/nexus/nexus_test.go index 8eb71159b97..47128e887c7 100644 --- a/common/rpc/interceptor/nexus/nexus_test.go +++ b/common/rpc/interceptor/nexus/nexus_test.go @@ -2,11 +2,35 @@ package nexus import ( "context" + "net/http" "testing" + "time" + "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" + "go.temporal.io/server/common/nexus/nexusrpc" ) +func TestInterceptorInputRequest(t *testing.T) { + dispatchRequest := &http.Request{Method: http.MethodPost} + requestStartTime := time.Date(2026, time.May, 5, 17, 0, 0, 123456789, time.UTC) + requestMetadata := RequestMetadata{Request: dispatchRequest} + inputs := []InterceptorInput{ + NewStartOpInput("s", "o", "n", requestStartTime, nexus.StartOperationOptions{}, nil, ForwardingInfo{}, requestMetadata), + NewCancelOpInput("s", "o", "n", requestStartTime, nexus.CancelOperationOptions{}, "t", ForwardingInfo{}, requestMetadata), + } + for _, input := range inputs { + require.Same(t, dispatchRequest, input.Request()) + require.True(t, input.StartTime().Equal(requestStartTime)) + } + + completionRequest := &nexusrpc.CompletionRequest{HTTPRequest: &http.Request{}} + completionInput, err := NewCompleteOpInput("n", requestStartTime, completionRequest, nil, ForwardingInfo{}, RequestMetadata{}) + require.NoError(t, err) + require.Same(t, completionRequest, completionInput.Request()) + require.True(t, completionInput.StartTime().Equal(requestStartTime)) +} + func TestChainNexusInterceptors(t *testing.T) { var calls []string chain := []Interceptor{ diff --git a/common/rpc/interceptor/rate_limit.go b/common/rpc/interceptor/rate_limit.go index e955f62dcd5..dccf8008b32 100644 --- a/common/rpc/interceptor/rate_limit.go +++ b/common/rpc/interceptor/rate_limit.go @@ -100,8 +100,9 @@ func (i *RateLimitInterceptor) InterceptNexus( ) (any, error) { if err := i.Allow(in.APIName(), in.Header()); err != nil { return nil, &nexus.InterceptorError{ - Err: err, - Outcome: "global_rate_limited", + Err: err, + Outcome: "global_rate_limited", + ExposeDetails: true, } } return next(ctx, in) diff --git a/common/rpc/interceptor/rate_limit_test.go b/common/rpc/interceptor/rate_limit_test.go index a2f1287a1a3..12d74beeae8 100644 --- a/common/rpc/interceptor/rate_limit_test.go +++ b/common/rpc/interceptor/rate_limit_test.go @@ -3,6 +3,7 @@ package interceptor import ( "context" "testing" + "time" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" @@ -31,29 +32,22 @@ func TestRateLimitInterceptorSuite(t *testing.T) { func (s *rateLimitInterceptorSuite) TestInterceptNexus() { for _, tc := range []struct { name string - apiName string input interceptornexus.InterceptorInput - allow *bool + allow bool nextCalled bool expectedOutcome string }{ - {name: "allowed", apiName: "NexusOperation", input: interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: new(true), nextCalled: true}, - {name: "rate limited", apiName: "NexusOperation", input: interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: new(false), expectedOutcome: "global_rate_limited"}, + {name: "allowed", input: interceptornexus.NewStartOpInput("service", "operation", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), allow: true, nextCalled: true}, + {name: "rate limited", input: interceptornexus.NewStartOpInput("service", "operation", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{APIName: "NexusOperation"}), expectedOutcome: "global_rate_limited"}, } { s.Run(tc.name, func() { ctx := context.Background() interceptor := NewRateLimitInterceptor(s.mockRateLimiter, nil) - if tc.allow != nil { - s.mockRateLimiter.EXPECT().Allow(gomock.Any(), gomock.Any()).Return(*tc.allow) - } - input := tc.input - if input == nil { - input = interceptornexus.NewStartOpInput("service", "operation", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}) - } + s.mockRateLimiter.EXPECT().Allow(gomock.Any(), gomock.Any()).Return(tc.allow) nextCalled := false _, err := interceptor.InterceptNexus( ctx, - input, + tc.input, func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil diff --git a/common/rpc/interceptor/sdk_version.go b/common/rpc/interceptor/sdk_version.go index e473e9ebde5..ea7683be22c 100644 --- a/common/rpc/interceptor/sdk_version.go +++ b/common/rpc/interceptor/sdk_version.go @@ -51,15 +51,15 @@ func (vi *SDKVersionInterceptor) InterceptNexus( in nexus.InterceptorInput, next nexus.HandlerFunc, ) (any, error) { - // draft-review: RecordSDKInfo didnt exist before, nice to add sdkName, sdkVersion := headers.GetClientNameAndVersion(ctx) if sdkName != "" && sdkVersion != "" { vi.RecordSDKInfo(sdkName, sdkVersion) } if err := vi.versionChecker.ClientSupported(ctx); err != nil { return nil, &nexus.InterceptorError{ - Err: err, - Outcome: "unsupported_client", + Err: err, + Outcome: "unsupported_client", + ExposeDetails: true, } } return next(ctx, in) diff --git a/common/rpc/interceptor/sdk_version_test.go b/common/rpc/interceptor/sdk_version_test.go index 5e6a9dac0f1..e0b40ace2f5 100644 --- a/common/rpc/interceptor/sdk_version_test.go +++ b/common/rpc/interceptor/sdk_version_test.go @@ -4,6 +4,7 @@ import ( "context" "sort" "testing" + "time" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/assert" @@ -102,7 +103,7 @@ func TestSDKVersionInterceptNexus(t *testing.T) { nextCalled := false _, err := interceptor.InterceptNexus( tc.ctx, - interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), + interceptornexus.NewStartOpInput("s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil diff --git a/common/rpc/interceptor/service_error_interceptor.go b/common/rpc/interceptor/service_error_interceptor.go index 41eb29d84c4..191bb73180e 100644 --- a/common/rpc/interceptor/service_error_interceptor.go +++ b/common/rpc/interceptor/service_error_interceptor.go @@ -4,6 +4,7 @@ import ( "context" "errors" + nexusrpc "github.com/nexus-rpc/sdk-go/nexus" "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" @@ -48,18 +49,17 @@ func (i *ServiceErrorInterceptor) Intercept( return resp, i.transformError(err) } -// InterceptNexus is a no-op: unlike the gRPC path, every error reaching the Nexus -// chain is already converted to a *nexus.HandlerError/*nexus.OperationError at its -// origin (see commonnexus.ConvertGRPCError call sites in nexus_handler.go and the -// other Nexus interceptors), so there's nothing left for transformError to do. This -// method exists only so ServiceErrorInterceptor keeps its chain position for parity -// with the gRPC ordering. func (i *ServiceErrorInterceptor) InterceptNexus( ctx context.Context, in nexus.InterceptorInput, next nexus.HandlerFunc, ) (any, error) { - return next(ctx, in) + resp, err := i.capturePanicHandlerNexus(ctx, in, next) + if ie, ok := errors.AsType[*nexus.InterceptorError](err); ok { + ie.Err = i.transformNexusError(ie.Err) + return resp, ie + } + return resp, i.transformNexusError(err) } func (i *ServiceErrorInterceptor) transformError(err error) error { @@ -92,3 +92,27 @@ func (i *ServiceErrorInterceptor) capturePanicHandler( defer metrics.CapturePanic(i.logger, i.metricsHandler, &retError) return handler(ctx, req) } + +func (i *ServiceErrorInterceptor) capturePanicHandlerNexus( + ctx context.Context, + in nexus.InterceptorInput, + next nexus.HandlerFunc, +) (_ any, retError error) { + defer metrics.CapturePanic(i.logger, i.metricsHandler, &retError) + return next(ctx, in) +} + +// transformNexusError only normalizes gRPC-shaped errors. Nexus-native errors +// are returned as-is to preserve existing mappings. +func (i *ServiceErrorInterceptor) transformNexusError(err error) error { + if err == nil { + return nil + } + if _, ok := errors.AsType[*nexusrpc.HandlerError](err); ok { + return err + } + if _, ok := errors.AsType[*nexusrpc.OperationError](err); ok { + return err + } + return i.transformError(err) +} diff --git a/common/rpc/interceptor/slow_request_logger.go b/common/rpc/interceptor/slow_request_logger.go index b07a2b8589e..e0fd88c0f3d 100644 --- a/common/rpc/interceptor/slow_request_logger.go +++ b/common/rpc/interceptor/slow_request_logger.go @@ -81,5 +81,5 @@ func (i *SlowRequestLoggerInterceptor) logSlowRequest( tags = append(tags, tag.Duration("duration", elapsed)) tags = append(tags, tag.String("method", method)) - i.logger.Warn("Slow gRPC call", tags...) + i.logger.Warn("Slow request", tags...) } diff --git a/common/rpc/interceptor/slow_request_logger_test.go b/common/rpc/interceptor/slow_request_logger_test.go index 35a01dfadae..9fc7625f937 100644 --- a/common/rpc/interceptor/slow_request_logger_test.go +++ b/common/rpc/interceptor/slow_request_logger_test.go @@ -73,7 +73,7 @@ func (s *slowRequestLoggerSuite) TestIntercept() { s.NoError(err) // Ensure slow requests are logged. - expectedMsg := "Slow gRPC call" + expectedMsg := "Slow request" s.logger.EXPECT().Warn(gomock.Eq(expectedMsg), gomock.Any()).Times(1) _, err = s.interceptor.Intercept(ctx, request, info, slowHandler) s.NoError(err) @@ -120,6 +120,7 @@ func (s *slowRequestLoggerSuite) TestInterceptNexus() { "test-service", "user-defined-operation", "namespace-name", + time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, @@ -131,7 +132,7 @@ func (s *slowRequestLoggerSuite) TestInterceptNexus() { s.Require().NoError(err) // Ensure slow requests are logged. - s.logger.EXPECT().Warn(gomock.Eq("Slow gRPC call"), gomock.Any()).Times(1) + s.logger.EXPECT().Warn(gomock.Eq("Slow request"), gomock.Any()).Times(1) _, err = s.interceptor.InterceptNexus(ctx, input, slowNext) s.Require().NoError(err) } diff --git a/common/rpc/interceptor/telemetry.go b/common/rpc/interceptor/telemetry.go index def718551f1..4011de82521 100644 --- a/common/rpc/interceptor/telemetry.go +++ b/common/rpc/interceptor/telemetry.go @@ -231,15 +231,16 @@ func (ti *TelemetryInterceptor) InterceptNexusOutermost( // chain (e.g. request forwarding) can still override the derived success outcome. ctx, outcomeOverride := nexus.NewOutcomeOverrideContext(ctx) - startTime := time.Now().UTC() - outcome, failed := nexus.OutcomeInternalError, true + startTime := in.StartTime() + outcome, failed := "internal_error", true + ctx = metrics.AddMetricsContext(ctx) defer func() { ti.RecordLatencyMetrics(ctx, startTime, serviceHandler) ti.recordNexusRequest(in, startTime, outcome, failed) }() out, err := next(ctx, in) - outcome, failed = nexus.Outcome(in, out, err), err != nil + outcome, failed = in.Outcome(out, err), err != nil // override outcome if its set - for request forwarding cases. // error cases are captured by the wrapped InterceptorError diff --git a/common/rpc/interceptor/telemetry_test.go b/common/rpc/interceptor/telemetry_test.go index 87af73e7854..5edae64d038 100644 --- a/common/rpc/interceptor/telemetry_test.go +++ b/common/rpc/interceptor/telemetry_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "time" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/assert" @@ -34,7 +35,7 @@ import ( func TestTelemetryInterceptNexusOutermost(t *testing.T) { extraTag := metrics.StringTag("configured", "tag") input := interceptornexus.NewStartOpInput( - "s", "o", testNamespace, nexus.StartOperationOptions{}, nil, + "s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{MetricTags: []metrics.Tag{extraTag}}, ) @@ -71,13 +72,13 @@ func TestTelemetryInterceptNexusOutermost(t *testing.T) { { name: "a short-circuiting interceptor overrides the success outcome", handlerOut: &nexus.HandlerStartOperationResultSync[any]{}, - setOverride: interceptornexus.OutcomeRequestForwarded, + setOverride: "request_forwarded", expectedOutcome: "request_forwarded", }, { name: "an error outcome wins over the override", handlerErr: &interceptornexus.InterceptorError{Err: errors.New("forward failed"), Outcome: "forwarded_request_error"}, - setOverride: interceptornexus.OutcomeRequestForwarded, + setOverride: "request_forwarded", expectedOutcome: "forwarded_request_error", expectedErrors: 1, }, @@ -139,7 +140,7 @@ func TestTelemetryInterceptNexusRecordsNothing(t *testing.T) { nextCalled := false _, err := telemetry.InterceptNexus( context.Background(), - interceptornexus.NewStartOpInput("s", "o", testNamespace, nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), + interceptornexus.NewStartOpInput("s", "o", testNamespace, time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), func(context.Context, interceptornexus.InterceptorInput) (any, error) { nextCalled = true return nil, nil diff --git a/service/frontend/frontend_interceptors.go b/service/frontend/frontend_interceptors.go index 2d5c948eada..b2342fd7682 100644 --- a/service/frontend/frontend_interceptors.go +++ b/service/frontend/frontend_interceptors.go @@ -9,8 +9,6 @@ import ( "go.temporal.io/server/common/rpc/grpcfaults" "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/common/rpc/interceptor/nexus" - "go.temporal.io/server/common/testing/grpcfaultstest" - "go.temporal.io/server/common/testing/testhooks" "google.golang.org/grpc" ) @@ -32,11 +30,8 @@ type Interceptor interface { } type InterceptorsProvider struct { - interceptors []Interceptor - nexusTelemetry nexus.Interceptor // required to be first in the Nexus chain - retryableInterceptor *interceptor.RetryableInterceptor // required to be last in chain after custom interceptors - customGRPCInterceptors []grpc.UnaryServerInterceptor // required for legacy reasons - faultGenerator grpcfaults.Generator + interceptors []Interceptor + nexusTelemetry nexus.Interceptor // required to be first in the Nexus chain } func NewInterceptorsProvider( @@ -46,10 +41,10 @@ func NewInterceptorsProvider( businessIDInterceptor *interceptor.RoutingKeyInterceptor, namespaceValidatorInterceptor *interceptor.NamespaceValidatorInterceptor, namespaceLogInterceptor *interceptor.NamespaceLogInterceptor, - metricsCtxInjectorInterceptor *metricsCtxInjectorInterceptor, authInterceptor *authorization.Interceptor, namespaceHandoverInterceptor *interceptor.NamespaceHandoverInterceptor, - redirectionSlot *redirectionWrapper, + redirectionInterceptor *interceptor.Redirection, + nexusForwarder *nexusForwardingInterceptor, telemetryInterceptor *interceptor.TelemetryInterceptor, healthInterceptor *interceptor.HealthInterceptor, namespaceStateValidatorInterceptor *interceptor.NamespaceStateValidatorInterceptor, @@ -63,10 +58,26 @@ func NewInterceptorsProvider( contextMetadataInterceptor *interceptor.ContextMetadataInterceptor, customGRPCInterceptors []grpc.UnaryServerInterceptor, customInterceptors []Interceptor, - testHooks testhooks.TestHooks, retryableInterceptor *interceptor.RetryableInterceptor, + faultsInterceptor *grpcfaults.FaultsInterceptor, ) *InterceptorsProvider { + metricsCtxInjectorInterceptor := &interceptorWrapper{ + grpcInterceptor: metrics.NewServerMetricsContextInjectorInterceptor(), + nexusInterceptor: nexusNoOpInterceptor, // added by telemetryInterceptor.InterceptNexusOutermost + } + + // redirectionWrapper is one chain position for both transports: gRPC DC redirection + // and Nexus HTTP forwarding. The implementations stay separate but are wrapped together + // for canonical ordering of interceptors for both gRPC and Nexus + redirectionWrapper := &interceptorWrapper{ + grpcInterceptor: redirectionInterceptor.Intercept, + nexusInterceptor: nexusForwarder.InterceptNexus, + } + + // Order is important. Error interceptors must stay outermost, routing must precede namespace + // access, and telemetry must follow redirection to attribute requests to the serving cluster. + // Nexus interceptors outward of error producers must preserve InterceptorError. interceptors := []Interceptor{ maskInternalErrorDetailsInterceptor, serviceErrorInterceptor, @@ -77,7 +88,7 @@ func NewInterceptorsProvider( metricsCtxInjectorInterceptor, authInterceptor, namespaceHandoverInterceptor, - redirectionSlot, + redirectionWrapper, telemetryInterceptor, healthInterceptor, namespaceValidatorInterceptor, @@ -90,37 +101,33 @@ func NewInterceptorsProvider( chasmRequestVisibilityInterceptor, contextMetadataInterceptor, } - // it is debatable if this should be *after* customGRPCInterceptors that are - // in use today. We will opt for this instead because relative ordering remains - // unchanged and anyone using customInterceptors should deprecate customGRPCInterceptors entirely + for _, grpcInterceptor := range customGRPCInterceptors { + interceptors = append(interceptors, &interceptorWrapper{ + grpcInterceptor: grpcInterceptor, + nexusInterceptor: nexusNoOpInterceptor, + }) + } interceptors = append(interceptors, customInterceptors...) + interceptors = append(interceptors, faultsInterceptor) + interceptors = append(interceptors, retryableInterceptor) + return &InterceptorsProvider{ - interceptors: interceptors, - nexusTelemetry: telemetryInterceptor.InterceptNexusOutermost, - customGRPCInterceptors: customGRPCInterceptors, - retryableInterceptor: retryableInterceptor, - faultGenerator: grpcfaultstest.NewGenerator(testHooks), + interceptors: interceptors, + nexusTelemetry: telemetryInterceptor.InterceptNexusOutermost, } } func (n *InterceptorsProvider) GrpcInterceptors() []grpc.UnaryServerInterceptor { - grpcInterceptors := make([]grpc.UnaryServerInterceptor, 0, len(n.interceptors)+len(n.customGRPCInterceptors)+1) + grpcInterceptors := make([]grpc.UnaryServerInterceptor, 0, len(n.interceptors)) for _, i := range n.interceptors { grpcInterceptors = append(grpcInterceptors, i.Intercept) } - // custom interceptors chain after system interceptors - grpcInterceptors = append(grpcInterceptors, n.customGRPCInterceptors...) - grpcInterceptors = append(grpcInterceptors, n.retryableInterceptor.Intercept) - - if faultInterceptor := grpcfaults.UnaryServerInterceptor(n.faultGenerator); faultInterceptor != nil { - grpcInterceptors = append(grpcInterceptors, faultInterceptor) - } return grpcInterceptors } func (n *InterceptorsProvider) NexusInterceptors() []nexus.Interceptor { - nexusInterceptors := make([]nexus.Interceptor, 0, len(n.interceptors)+2) + nexusInterceptors := make([]nexus.Interceptor, 0, len(n.interceptors)+1) // telemetry is the outermost in chain for Nexus requests to allow recording // all metrics and retain behavior. In the future, gRPC will also move telemetry // to outermost after an impact evaluation- this will allow gRPC to also capture @@ -129,55 +136,35 @@ func (n *InterceptorsProvider) NexusInterceptors() []nexus.Interceptor { for _, i := range n.interceptors { nexusInterceptors = append(nexusInterceptors, i.InterceptNexus) } - - nexusInterceptors = append(nexusInterceptors, n.retryableInterceptor.InterceptNexus) return nexusInterceptors } -// redirectionWrapper is one chain position for both transports: gRPC DC redirection -// and Nexus HTTP forwarding. The implementations stay separate but are wrapped together -// for canonical ordering of interceptors for both gRPC and Nexus -type redirectionWrapper struct { - grpc *interceptor.Redirection - nexus *nexusForwardingInterceptor +type interceptorWrapper struct { + grpcInterceptor grpc.UnaryServerInterceptor + nexusInterceptor nexus.Interceptor } -func (s *redirectionWrapper) Intercept( +func (i interceptorWrapper) Intercept( ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ) (any, error) { - return s.grpc.Intercept(ctx, req, info, handler) + return i.grpcInterceptor(ctx, req, info, handler) } -func (s *redirectionWrapper) InterceptNexus( +func (i interceptorWrapper) InterceptNexus( ctx context.Context, in nexus.InterceptorInput, next nexus.HandlerFunc, ) (any, error) { - return s.nexus.InterceptNexus(ctx, in, next) -} - -// tiny wrapper to inject metrics context and avoid -// cyclical dependencies in metrics/interceptors packages -type metricsCtxInjectorInterceptor struct{} - -func (m *metricsCtxInjectorInterceptor) Intercept( - ctx context.Context, - req any, - info *grpc.UnaryServerInfo, - handler grpc.UnaryHandler, -) (any, error) { - ctxWithMetricsBaggage := metrics.AddMetricsContext(ctx) - return handler(ctxWithMetricsBaggage, req) + return i.nexusInterceptor(ctx, in, next) } -func (m *metricsCtxInjectorInterceptor) InterceptNexus( +func nexusNoOpInterceptor( ctx context.Context, in nexus.InterceptorInput, next nexus.HandlerFunc, ) (any, error) { - ctxWithMetricsBaggage := metrics.AddMetricsContext(ctx) - return next(ctxWithMetricsBaggage, in) + return next(ctx, in) } diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 0d7461c8448..62a3dd6327f 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -43,10 +43,12 @@ import ( "go.temporal.io/server/common/resource" "go.temporal.io/server/common/rpc" "go.temporal.io/server/common/rpc/encryption" + "go.temporal.io/server/common/rpc/grpcfaults" "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/common/sdk" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/telemetry" + "go.temporal.io/server/common/testing/grpcfaultstest" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service" "go.temporal.io/server/service/frontend/configs" @@ -96,8 +98,6 @@ var Module = fx.Options( fx.Provide(interceptor.NewRoutingKeyExtractor), fx.Provide(BusinessIDInterceptorProvider), fx.Provide(RedirectionInterceptorProvider), - fx.Provide(RedirectionSlotProvider), - fx.Provide(NewMetricsContextInjectorInterceptor), fx.Provide(ErrorHandlerProvider), fx.Provide(TelemetryInterceptorProvider), fx.Provide(RetryableInterceptorProvider), @@ -134,6 +134,7 @@ var Module = fx.Options( fx.Provide(ServiceResolverProvider), fx.Provide(newNexusForwardingInterceptor), fx.Provide(interceptor.NewNamespaceRateLimitInterceptorWrapper), + fx.Provide(NewFaultsInterceptorProvider), fx.Provide(NewInterceptorsProvider), fx.Provide(newNexusCompletionHandler), fx.Provide(NewNexusOperationHTTPHandler), @@ -238,7 +239,6 @@ func (n *namespaceChecker) Exists(name namespace.Name) error { func GrpcServerOptionsProvider( logger log.Logger, - cfg *config.Config, serviceConfig *Config, serviceName primitives.ServiceName, rpcFactory common.RPCFactory, @@ -248,8 +248,6 @@ func GrpcServerOptionsProvider( metricsStatsHandler metrics.ServerStatsHandler, authInterceptor *authorization.Interceptor, customStreamInterceptors []grpc.StreamServerInterceptor, - metricsHandler metrics.Handler, - testHooks testhooks.TestHooks, ) GrpcServerOptions { kep := keepalive.EnforcementPolicy{ MinTime: serviceConfig.KeepAliveMinTime(), @@ -349,20 +347,6 @@ func RetryableInterceptorProvider() *interceptor.RetryableInterceptor { ) } -func RedirectionSlotProvider( - redirectionInterceptor *interceptor.Redirection, - nexusForwarder *nexusForwardingInterceptor, -) *redirectionWrapper { - return &redirectionWrapper{ - grpc: redirectionInterceptor, - nexus: nexusForwarder, - } -} - -func NewMetricsContextInjectorInterceptor() *metricsCtxInjectorInterceptor { - return &metricsCtxInjectorInterceptor{} -} - func RedirectionInterceptorProvider( configuration *Config, namespaceCache namespace.Registry, @@ -759,6 +743,12 @@ func FEReplicatorNamespaceReplicationQueueProvider( return replicatorNamespaceReplicationQueue } +func NewFaultsInterceptorProvider(hooks testhooks.TestHooks) *grpcfaults.FaultsInterceptor { + return grpcfaults.NewFaultsInterceptor( + grpcfaultstest.NewGenerator(hooks), + ) +} + func ServiceResolverProvider( membershipMonitor membership.Monitor, serviceName primitives.ServiceName, diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index 52f3ce8f963..003075c5be7 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -3,9 +3,12 @@ package frontend import ( "context" "errors" + "fmt" "net/http" "net/url" + "runtime/debug" "strings" + "time" "github.com/gorilla/mux" "github.com/nexus-rpc/sdk-go/nexus" @@ -38,7 +41,6 @@ const nexusCompletionAPIName = configs.CompleteNexusOperation const nexusCompletionMethodName = "CompleteNexusOperation" type nexusCompletionHandler struct { - ClusterMetadata cluster.Metadata NamespaceRegistry namespace.Registry Logger log.Logger MetricsHandler metrics.Handler @@ -47,8 +49,6 @@ type nexusCompletionHandler struct { HistoryClient resource.HistoryClient RequestErrorHandler *interceptor.RequestErrorHandler AuthInterceptor *authorization.Interceptor // required for parsing auth info, not used as an interceptor - HTTPTraceProvider commonnexus.HTTPClientTraceProvider - clientVersionChecker headers.VersionChecker preProcessErrorsCounter metrics.CounterIface chainedHandler interceptornexus.HandlerFunc } @@ -72,7 +72,6 @@ func newNexusCompletionHandler( ) *nexusCompletionHandler { h := &nexusCompletionHandler{ - ClusterMetadata: clusterMetadata, NamespaceRegistry: namespaceRegistry, Logger: log.With(logger, tag.NexusStageCallerInbound), MetricsHandler: metricsHandler, @@ -81,8 +80,6 @@ func newNexusCompletionHandler( HistoryClient: historyClient, RequestErrorHandler: requestErrorHandler, AuthInterceptor: authInterceptor, - HTTPTraceProvider: httpTraceProvider, - clientVersionChecker: headers.NewDefaultVersionChecker(), preProcessErrorsCounter: metricsHandler.Counter(metrics.NexusCompletionRequestPreProcessErrors.Name()), } h.chainedHandler = interceptornexus.ChainInterceptors(h.finalCompleteHandler, interceptorsProvider.NexusInterceptors()) @@ -102,6 +99,7 @@ func newNexusCompletionHTTPHandler(handler *nexusCompletionHandler) *nexusComple // CompleteOperation implements nexus.CompletionHandler. // nolint:revive // (cyclomatic complexity) This function is long but the complexity is justified. func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexusrpc.CompletionRequest) (retErr error) { + requestStartTime := time.Now() token, err := commonnexus.DecodeCallbackToken(r.HTTPRequest.Header.Get(commonnexus.CallbackTokenHeader)) if err != nil { h.Logger.Error("failed to decode callback token", tag.Error(err)) @@ -144,7 +142,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus rCtx := &requestContext{ nexusCompletionHandler: h, namespace: ns, - logger: log.With(h.Logger, tag.WorkflowNamespace(ns.Name().String())), + logger: logger, metricsHandlerForInterceptors: h.MetricsHandler.WithTags( metrics.OperationTag(nexusCompletionMethodName), metrics.NamespaceTag(ns.Name().String()), @@ -156,29 +154,51 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus ctx = rCtx.augmentContext(ctx, r.HTTPRequest.Header) defer finalizeCompletionRequest(rCtx, &retErr) + // recordBadRequest is for pre-interceptor chain error recording + recordBadRequest := func() { + metrics.NexusCompletionRequests.With(h.MetricsHandler).Record( + 1, + metrics.NamespaceTag(ns.Name().String()), + metrics.OutcomeTag("error_bad_request"), + ) + } + if r.HTTPRequest.URL.Path != commonnexus.PathCompletionCallbackNoIdentifier { nsNameEscaped := commonnexus.RouteCompletionCallback.Deserialize(mux.Vars(r.HTTPRequest)) nsName, err := url.PathUnescape(nsNameEscaped) if err != nil { logger.Error("failed to extract namespace from request", tag.Error(err)) h.preProcessErrorsCounter.Record(1) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid URL") + recordBadRequest() + return &interceptornexus.InterceptorError{ + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid URL"), + SkipServiceErrorReporting: true, + } } if nsName != ns.Name().String() { logger.Error( "namespace in callback URL doesn't match the completion token", tag.String("url-namespace", nsName), ) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid callback token") + recordBadRequest() + return &interceptornexus.InterceptorError{ + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid callback token"), + SkipServiceErrorReporting: true, + } } } ctx, err = rCtx.parseTLSAndAuthInfo(ctx, r) if err != nil { - return err + recordBadRequest() + return &interceptornexus.InterceptorError{ + Err: err, + SkipServiceErrorReporting: true, + } } interceptorInput, err := interceptornexus.NewCompleteOpInput( ns.Name().String(), + requestStartTime, r, completion, interceptornexus.ForwardingInfo{ @@ -192,7 +212,11 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus ) if err != nil { logger.Error("invalid nexus completion request", tag.Error(err)) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid request") + recordBadRequest() + return &interceptornexus.InterceptorError{ + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid request"), + SkipServiceErrorReporting: true, + } } ctx = withRequestContext(ctx, rCtx) _, err = h.chainedHandler(ctx, interceptorInput) @@ -252,7 +276,7 @@ func (h *nexusCompletionHandler) finalCompleteHandler( return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive") } if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { - return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "error_not_found"} + return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "error_not_found", ExposeDetails: true} } return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "error_internal"} } @@ -467,11 +491,12 @@ func (c *requestContext) handleRequestError(err error) { return } if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { + if taggedErr.SkipServiceErrorReporting { + return + } err = taggedErr.Err } c.RequestErrorHandler.HandleError( - // The request is only read to extract workflow log tags, which is keyed off the - // gRPC full method. Nexus has none, so it is never used. nil, "", c.metricsHandlerForInterceptors, @@ -485,7 +510,14 @@ func (c *requestContext) handleRequestError(err error) { // panic into errPtr, log/classify the (still raw) resulting error, then sanitize it for the // response. Order matters and must not be split back into separate defers. func finalizeCompletionRequest(rCtx *requestContext, errPtr *error) { - captureOperationPanic(rCtx.logger, errPtr) + if recovered := recover(); recovered != nil { //nolint:revive + err, ok := recovered.(error) + if !ok { + err = fmt.Errorf("panic: %v", recovered) + } + rCtx.logger.Error("Panic captured", tag.SysStackTrace(string(debug.Stack())), tag.Error(err)) + *errPtr = err + } rCtx.handleRequestError(*errPtr) *errPtr = convertInterceptorError(*errPtr) } diff --git a/service/frontend/nexus_dispatch_result.go b/service/frontend/nexus_dispatch_result.go index ad76571d891..3bb104cfb4a 100644 --- a/service/frontend/nexus_dispatch_result.go +++ b/service/frontend/nexus_dispatch_result.go @@ -7,6 +7,7 @@ import ( "go.temporal.io/server/common/log/tag" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" ) // handleStartOperationResponse converts matching's response to a StartOperation dispatch into the result the @@ -38,16 +39,16 @@ func (c *operationContext) handleStartOperationResponse( // answer, reported to the caller as a Nexus operation error rather than a handler error. cause, internalErr := c.convertWorkerFailure(result.Failure, operation) if internalErr != nil { - return nil, nil, internalErr + return nil, nil, dispatchError(result, internalErr) } state := nexus.OperationStateFailed if result.Failure.GetCanceledFailureInfo() != nil { state = nexus.OperationStateCanceled } - return nil, nil, c.operationError(state, cause, operation) + return nil, nil, dispatchError(result, c.operationError(state, cause, operation)) default: - return nil, nil, c.failedDispatchToNexusError(result, operation) + return nil, nil, dispatchError(result, c.failedDispatchToNexusError(result, operation)) } } @@ -64,7 +65,19 @@ func (c *operationContext) handleCancelOperationResponse( if result.Outcome == commonnexus.DispatchOutcomeCancelAccepted { return nil } - return c.failedDispatchToNexusError(result, operation) + return dispatchError(result, c.failedDispatchToNexusError(result, operation)) +} + +// dispatchError wraps the error with the result's outcome tag so it can +// be tagged in turn by the nexus telemetry outermost interceptor +func dispatchError(result commonnexus.DispatchResult, err error) error { + if err == nil { + return nil + } + return &interceptornexus.InterceptorError{ + Err: err, + Outcome: result.OutcomeTag().Value, + } } // failedDispatchToNexusError converts the outcomes that mean the task was never handled, or was @@ -140,10 +153,9 @@ func (c *operationContext) operationError( return opErr } -// recordDispatchOutcome tags the request's metrics with the dispatch outcome and, when the dispatch -// did not succeed, attributes the failure to the worker in the response header. +// recordDispatchOutcome attributes a failed dispatch to the worker in the response header. The +// outcome is carried by the returned InterceptorError and recorded by the Nexus telemetry interceptor. func (c *operationContext) recordDispatchOutcome(result commonnexus.DispatchResult) { - c.metricsHandler = c.metricsHandler.WithTags(result.OutcomeTag()) if !result.Outcome.Succeeded() { c.setFailureSource(commonnexus.FailureSourceWorker) } diff --git a/service/frontend/nexus_dispatch_result_test.go b/service/frontend/nexus_dispatch_result_test.go index 093e038d3b4..ecea95228f5 100644 --- a/service/frontend/nexus_dispatch_result_test.go +++ b/service/frontend/nexus_dispatch_result_test.go @@ -1,8 +1,10 @@ package frontend import ( + "context" "encoding/json" "testing" + "time" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" @@ -11,8 +13,12 @@ import ( failurepb "go.temporal.io/api/failure/v1" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/metrics/metricstest" commonnexus "go.temporal.io/server/common/nexus" + rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" ) // These tests pin down how the frontend turns matching's DispatchNexusTaskResponse into the result the @@ -21,30 +27,46 @@ import ( // dashboards and by interceptRequest's error-reporting cleanup. They are asserted here so the shared // classifier introduced alongside them cannot silently change any of it. -// outcomeTagOf reads the outcome tag accumulated on the context's metrics handler. -func outcomeTagOf(t *testing.T, oc *operationContext) string { +func failureSourceOf(oc *operationContext) string { + return oc.responseHeaders[commonnexus.FailureSourceHeaderName] +} + +func requireDispatchOutcome(t *testing.T, err error, outcome string) { t.Helper() - mh, ok := oc.metricsHandler.(*metricstest.CaptureHandler) - require.True(t, ok, "expected a capture handler") - capture := mh.StartCapture() - oc.metricsHandler.Counter("test").Record(1) - mh.StopCapture(capture) - snap := capture.Snapshot() - require.Len(t, snap["test"], 1) - return snap["test"][0].Tags["outcome"] + var interceptorErr *interceptornexus.InterceptorError + require.ErrorAs(t, err, &interceptorErr) + require.Equal(t, outcome, interceptorErr.Outcome) } -func failureSourceOf(oc *operationContext) string { - return oc.responseHeaders[commonnexus.FailureSourceHeaderName] +func requireRecordedDispatchOutcome( + t *testing.T, + input interceptornexus.InterceptorInput, + expectedOutcome string, + handler func(*operationContext) error, +) { + t.Helper() + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + + telemetry := rpcinterceptor.NewTelemetryInterceptor(nil, metricsHandler, log.NewNoopLogger(), nil, nil) + _, err := telemetry.InterceptNexusOutermost( + context.Background(), + input, + func(context.Context, interceptornexus.InterceptorInput) (any, error) { + return nil, handler(testOperationContext()) + }, + ) + requireDispatchOutcome(t, err, expectedOutcome) + + snapshot := capture.Snapshot() + require.Len(t, snapshot[metrics.NexusRequests.Name()], 1) + outcomeTag := metrics.OutcomeTag(expectedOutcome) + require.Equal(t, expectedOutcome, snapshot[metrics.NexusRequests.Name()][0].Tags[outcomeTag.Key]) } func testOperationContext() *operationContext { - return newOperationContext(contextOptions{ - namespaceState: enumspb.NAMESPACE_STATE_REGISTERED, - quota: 1, - namespaceRateLimitAllow: true, - rateLimitAllow: true, - }) + return newOperationContext() } // startOperationResponse wraps a StartOperationResponse in the matching response envelope. The oneof @@ -59,6 +81,78 @@ func startOperationResponse(sor *nexuspb.StartOperationResponse) *matchingservic } } +func TestDispatchErrorsPreserveOutcomeForTelemetry(t *testing.T) { + handlerFailure := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: string(nexus.HandlerErrorTypeBadRequest), + }, + }, + }, + }, + } + requestTimeout := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_RequestTimeout{ + RequestTimeout: &matchingservice.DispatchNexusTaskResponse_Timeout{}, + }, + } + operationFailure := startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_Failure{ + Failure: &failurepb.Failure{ + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{}, + }, + }, + }, + }) + + for _, tc := range []struct { + name string + response *matchingservice.DispatchNexusTaskResponse + outcome string + }{ + {name: "handler failure", response: handlerFailure, outcome: "handler_error:BAD_REQUEST"}, + {name: "request timeout", response: requestTimeout, outcome: "handler_timeout"}, + {name: "operation failure", response: operationFailure, outcome: "failure"}, + {name: "unrecognized outcome", response: &matchingservice.DispatchNexusTaskResponse{}, outcome: "handler_error:EMPTY_OUTCOME"}, + } { + t.Run("start "+tc.name, func(t *testing.T) { + requireRecordedDispatchOutcome( + t, + interceptornexus.NewStartOpInput("s", "o", "n", time.Now(), nexus.StartOperationOptions{}, nil, interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), + tc.outcome, + func(oc *operationContext) error { + _, _, err := oc.handleStartOperationResponse(tc.response, "op") + return err + }, + ) + }) + } + + for _, tc := range []struct { + name string + response *matchingservice.DispatchNexusTaskResponse + outcome string + }{ + {name: "handler failure", response: handlerFailure, outcome: "handler_error:BAD_REQUEST"}, + {name: "request timeout", response: requestTimeout, outcome: "handler_timeout"}, + {name: "unrecognized outcome", response: &matchingservice.DispatchNexusTaskResponse{}, outcome: "handler_error:EMPTY_OUTCOME"}, + } { + t.Run("cancel "+tc.name, func(t *testing.T) { + requireRecordedDispatchOutcome( + t, + interceptornexus.NewCancelOpInput("s", "o", "n", time.Now(), nexus.CancelOperationOptions{}, "t", interceptornexus.ForwardingInfo{}, interceptornexus.RequestMetadata{}), + tc.outcome, + func(oc *operationContext) error { + return oc.handleCancelOperationResponse(tc.response, "op") + }, + ) + }) + } +} + func TestHandleStartOperationResponse_SyncSuccess(t *testing.T) { oc := testOperationContext() payload := &commonpb.Payload{Data: []byte("hello")} @@ -82,7 +176,6 @@ func TestHandleStartOperationResponse_SyncSuccess(t *testing.T) { require.Len(t, links, 1) require.Equal(t, "http://links.test/valid", links[0].URL.String()) require.Equal(t, "some.Type", links[0].Type) - require.Equal(t, "sync_success", outcomeTagOf(t, oc)) require.Empty(t, failureSourceOf(oc), "success must not be attributed to the worker") } @@ -100,7 +193,6 @@ func TestHandleStartOperationResponse_SyncSuccess_NoPayloadNoLinks(t *testing.T) require.True(t, ok) require.Nil(t, sync.Value) require.Empty(t, links) - require.Equal(t, "sync_success", outcomeTagOf(t, oc)) } func TestHandleStartOperationResponse_AsyncSuccess_PrefersOperationToken(t *testing.T) { @@ -122,7 +214,6 @@ func TestHandleStartOperationResponse_AsyncSuccess_PrefersOperationToken(t *test require.True(t, ok, "expected an async result, got %T", result) require.Equal(t, "token", async.OperationToken) require.Len(t, links, 1) - require.Equal(t, "async_success", outcomeTagOf(t, oc)) require.Empty(t, failureSourceOf(oc)) } @@ -191,7 +282,7 @@ func TestHandleStartOperationResponse_HandlerFailure(t *testing.T) { require.Equal(t, "handler said no", handlerErr.Message) require.Equal(t, tc.wantRetryable, handlerErr.Retryable()) require.NoError(t, handlerErr.Cause, "no cause on the wire means no cause on the error") - require.Equal(t, "handler_error:BAD_REQUEST", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_error:BAD_REQUEST") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) }) } @@ -255,7 +346,7 @@ func TestHandleStartOperationResponse_WorkerFailure_NotAHandlerError(t *testing. var handlerErr *nexus.HandlerError require.NotErrorAs(t, err, &handlerErr, "not reported as a handler error today") // There is no handler error type to report, so the tag bounds to UNKNOWN. - require.Equal(t, "handler_error:UNKNOWN", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_error:UNKNOWN") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -282,7 +373,7 @@ func TestHandleStartOperationResponse_DeprecatedHandlerError(t *testing.T) { deprecatedCause, ok := handlerErr.Cause.(*nexus.FailureError) require.True(t, ok, "expected a Nexus FailureError cause, got %T", handlerErr.Cause) require.Equal(t, "slow down", deprecatedCause.Failure.Message) - require.Equal(t, "handler_error:RESOURCE_EXHAUSTED", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_error:RESOURCE_EXHAUSTED") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -301,7 +392,7 @@ func TestHandleStartOperationResponse_RequestTimeout(t *testing.T) { require.ErrorAs(t, err, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeUpstreamTimeout, handlerErr.Type) require.Equal(t, "upstream timeout", handlerErr.Message) - require.Equal(t, "handler_timeout", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_timeout") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -351,7 +442,7 @@ func TestHandleStartOperationResponse_OperationFailure(t *testing.T) { require.NotNil(t, opErr.OriginalFailure) require.Equal(t, "true", opErr.OriginalFailure.Metadata["unwrap-error"]) require.NotNil(t, opErr.OriginalFailure.Cause) - require.Equal(t, "failure", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "failure") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) }) } @@ -386,7 +477,7 @@ func TestHandleStartOperationResponse_OperationFailure_UnconvertibleFailureIsInt var opErr *nexus.OperationError require.NotErrorAs(t, err, &opErr, "an unreadable failure is not a legitimate operation error") // The outcome was still classified as an operation failure, so the tag and header stand. - require.Equal(t, "failure", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "failure") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -415,7 +506,7 @@ func TestHandleStartOperationResponse_HandlerFailure_UnconvertibleCauseIsInterna require.ErrorAs(t, err, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type, "the worker's own BAD_REQUEST must not survive a failed conversion") - require.Equal(t, "handler_error:BAD_REQUEST", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_error:BAD_REQUEST") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -459,7 +550,7 @@ func TestHandleStartOperationResponse_DeprecatedOperationError(t *testing.T) { require.Equal(t, "worker canceled it", cause.Failure.Message) require.NotNil(t, opErr.OriginalFailure) require.Equal(t, "true", opErr.OriginalFailure.Metadata["unwrap-error"]) - require.Equal(t, "operation_error", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "operation_error") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -502,7 +593,7 @@ func TestHandleStartOperationResponse_DeprecatedOperationErrorReEncodesWorkerFai require.NoError(t, json.Unmarshal(details[0].GetData(), &workerFailure)) require.Equal(t, map[string]string{"k": "v"}, workerFailure.Metadata) require.JSONEq(t, `"details"`, string(workerFailure.Details)) - require.Equal(t, "operation_error", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "operation_error") } // Anything the frontend cannot interpret is blamed on the worker and reported as an internal error. @@ -545,7 +636,7 @@ func TestHandleStartOperationResponse_UnrecognizedOutcomes(t *testing.T) { require.ErrorAs(t, err, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type) require.Equal(t, "empty outcome", handlerErr.Message) - require.Equal(t, "handler_error:EMPTY_OUTCOME", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_error:EMPTY_OUTCOME") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) }) } @@ -581,7 +672,6 @@ func TestHandleCancelOperationResponse_Success(t *testing.T) { t.Run(tc.name, func(t *testing.T) { oc := testOperationContext() require.NoError(t, oc.handleCancelOperationResponse(tc.resp, "op")) - require.Equal(t, "success", outcomeTagOf(t, oc)) require.Empty(t, failureSourceOf(oc)) }) } @@ -607,7 +697,7 @@ func TestHandleCancelOperationResponse_HandlerFailure(t *testing.T) { require.ErrorAs(t, err, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeNotFound, handlerErr.Type) require.Equal(t, "cannot cancel", handlerErr.Message) - require.Equal(t, "handler_error:NOT_FOUND", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_error:NOT_FOUND") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -627,7 +717,7 @@ func TestHandleCancelOperationResponse_DeprecatedHandlerError(t *testing.T) { var handlerErr *nexus.HandlerError require.ErrorAs(t, err, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeNotImplemented, handlerErr.Type) - require.Equal(t, "handler_error:NOT_IMPLEMENTED", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_error:NOT_IMPLEMENTED") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -644,7 +734,7 @@ func TestHandleCancelOperationResponse_RequestTimeout(t *testing.T) { require.ErrorAs(t, err, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeUpstreamTimeout, handlerErr.Type) require.Equal(t, "upstream timeout", handlerErr.Message) - require.Equal(t, "handler_timeout", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_timeout") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -655,7 +745,7 @@ func TestHandleCancelOperationResponse_UnrecognizedOutcome(t *testing.T) { require.ErrorAs(t, err, &handlerErr) require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type) require.Equal(t, "empty outcome", handlerErr.Message) - require.Equal(t, "handler_error:EMPTY_OUTCOME", outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, "handler_error:EMPTY_OUTCOME") require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) } @@ -725,7 +815,7 @@ func TestHandleStartOperationResponse_HandlerErrorTypeTagIsBounded(t *testing.T) _, _, err := oc.handleStartOperationResponse(resp, "op") require.Error(t, err) - require.Equal(t, tc.wantTag, outcomeTagOf(t, oc)) + requireDispatchOutcome(t, err, tc.wantTag) // The error itself still carries the worker's real type; only the metric is bounded. var handlerErr *nexus.HandlerError require.ErrorAs(t, err, &handlerErr) @@ -744,6 +834,7 @@ func TestHandleCancelOperationResponse_DeprecatedHandlerErrorTypeTagIsBounded(t }, } - require.Error(t, oc.handleCancelOperationResponse(resp, "op")) - require.Equal(t, "handler_error:UNKNOWN", outcomeTagOf(t, oc)) + err := oc.handleCancelOperationResponse(resp, "op") + require.Error(t, err) + requireDispatchOutcome(t, err, "handler_error:UNKNOWN") } diff --git a/service/frontend/nexus_forward_interceptor.go b/service/frontend/nexus_forward_interceptor.go index b7a86ec0734..f6a768b594a 100644 --- a/service/frontend/nexus_forward_interceptor.go +++ b/service/frontend/nexus_forward_interceptor.go @@ -63,8 +63,9 @@ func (i *nexusForwardingInterceptor) InterceptNexus( namespaceEntry, err := in.NamespaceEntry() if err != nil { return nil, &interceptornexus.InterceptorError{ - Err: err, - Outcome: "interceptor_failed", + Err: err, + Outcome: "interceptor_failed", + SkipServiceErrorReporting: true, } } currentCluster := i.clusterMetadata.GetCurrentClusterName() @@ -74,12 +75,13 @@ func (i *nexusForwardingInterceptor) InterceptNexus( } if !i.shouldForwardRequest(ctx, header, namespaceEntry) { return nil, &interceptornexus.InterceptorError{ - Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive"), - Outcome: "namespace_inactive_forwarding_disabled", + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive"), + Outcome: "namespace_inactive_forwarding_disabled", + SkipServiceErrorReporting: true, } } - interceptornexus.SetOutcomeOverride(ctx, interceptornexus.OutcomeRequestForwarded) + interceptornexus.SetOutcomeOverride(ctx, "request_forwarded") // this is the user-facing operation identity, and the DCRedirection prefix // matches the convention the gRPC redirection path uses for the same metrics. @@ -94,16 +96,33 @@ func (i *nexusForwardingInterceptor) InterceptNexus( i.redirectionInterceptor.AfterCall(metricsHandler, forwardStartTime, targetCluster, namespaceEntry.Name().String(), redirectionErr) }() + logTags := []tag.Tag{ + tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), + tag.TargetCluster(targetCluster), + tag.Operation(in.MethodName()), + tag.WorkflowNamespace(namespaceEntry.Name().String()), + } + if endpointName := in.EndpointName(); endpointName != "" { + // empty on namespace/task-queue routed requests + logTags = append(logTags, tag.Endpoint(endpointName)) + } + if operationName := in.OperationName(); operationName != "" { + // empty for completion requests + logTags = append(logTags, tag.NexusOperation(operationName)) + } + logger := log.With(i.logger, logTags...) + switch request := in.(type) { case interceptornexus.StartOpInput: - out, retErr = i.forwardStartOperation(ctx, request, info, namespaceEntry, targetCluster) + out, retErr = i.forwardStartOperation(ctx, logger, request, info, namespaceEntry, targetCluster) case interceptornexus.CancelOpInput: - retErr = i.forwardCancelOperation(ctx, request, info, namespaceEntry, targetCluster) + retErr = i.forwardCancelOperation(ctx, logger, request, info, namespaceEntry, targetCluster) case interceptornexus.CompleteOpInput: - retErr = i.forwardCompleteOperation(ctx, request, info, namespaceEntry, targetCluster) + retErr = i.forwardCompleteOperation(ctx, logger, request, info, namespaceEntry, targetCluster) default: return nil, &interceptornexus.InterceptorError{ - Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "forwarding failed, unknown operation type"), + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "forwarding failed, unknown operation type"), + SkipServiceErrorReporting: true, } } return out, retErr @@ -120,25 +139,24 @@ func (i *nexusForwardingInterceptor) shouldForwardRequest( } return redirectAllowed && i.redirectionInterceptor.RedirectionAllowed(ctx) && - namespaceEntry.IsGlobalNamespace() && i.serviceConfig.EnableNamespaceNotActiveAutoForwarding(namespaceEntry.Name().String()) } func (i *nexusForwardingInterceptor) forwardStartOperation( ctx context.Context, + logger log.Logger, request interceptornexus.StartOpInput, info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, ) (any, error) { - logger := log.With( - i.logger, - tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), - tag.TargetCluster(targetCluster), + logger = log.With( + logger, + tag.RequestID(request.StartOperationOptions.RequestID), ) request.StartOperationOptions.Header[interceptor.DCRedirectionAPIHeaderName] = "true" request.StartOperationOptions.Header[interceptor.DCRedirectionSourceCellHeaderName] = i.clusterMetadata.GetCurrentClusterName() - client, err := i.nexusClientForActiveCluster(ctx, request.ServiceName(), info, namespaceEntry, targetCluster) + client, err := i.nexusClientForActiveCluster(ctx, logger, request.ServiceName(), info, namespaceEntry, targetCluster) if err != nil { return nil, err } @@ -146,7 +164,7 @@ func (i *nexusForwardingInterceptor) forwardStartOperation( response, err := client.StartOperation(ctx, request.OperationName(), request.StartOperationInput.Reader, request.StartOperationOptions) if err != nil { logger.Error("received error from remote cluster for forwarded Nexus start operation request", tag.Error(err)) - return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error"} + return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error", SkipServiceErrorReporting: true} } if response.Successful != nil { return &nexus.HandlerStartOperationResultSync[any]{Value: response.Successful.Reader}, nil @@ -156,64 +174,57 @@ func (i *nexusForwardingInterceptor) forwardStartOperation( func (i *nexusForwardingInterceptor) forwardCancelOperation( ctx context.Context, + logger log.Logger, request interceptornexus.CancelOpInput, info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, ) error { - logger := log.With( - i.logger, - tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), - tag.TargetCluster(targetCluster), - ) request.CancelOperationOptions.Header[interceptor.DCRedirectionAPIHeaderName] = "true" request.CancelOperationOptions.Header[interceptor.DCRedirectionSourceCellHeaderName] = i.clusterMetadata.GetCurrentClusterName() - client, err := i.nexusClientForActiveCluster(ctx, request.ServiceName(), info, namespaceEntry, targetCluster) + client, err := i.nexusClientForActiveCluster(ctx, logger, request.ServiceName(), info, namespaceEntry, targetCluster) if err != nil { return err } handle, err := client.NewOperationHandle(request.OperationName(), request.CancellationToken) if err != nil { logger.Warn("invalid Nexus cancel operation", tag.Error(err)) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation") + return &interceptornexus.InterceptorError{ + Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation"), + Outcome: "error_bad_request", + } } ctx = i.withForwardingTrace(ctx, "CancelNexusOperation", request.OperationName(), "", info, namespaceEntry, targetCluster) if err := handle.Cancel(ctx, request.CancelOperationOptions); err != nil { logger.Error("received error from remote cluster for forwarded Nexus cancel operation request", tag.Error(err)) - return &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error"} + return &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error", SkipServiceErrorReporting: true} } return nil } func (i *nexusForwardingInterceptor) forwardCompleteOperation( ctx context.Context, + logger log.Logger, request interceptornexus.CompleteOpInput, info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, targetCluster string, ) error { - logger := log.With( - i.logger, - tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), - tag.TargetCluster(targetCluster), - ) client, err := i.forwardingClients.Get(targetCluster) if err != nil { - logger.Error("unable to get HTTP client for forward request", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) - return &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} + logger.Error("unable to get HTTP client for forward request", tag.Error(err)) + return &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed", SkipServiceErrorReporting: true} } forwardURL, err := url.JoinPath(client.BaseURL(), commonnexus.RouteCompletionCallback.Path(namespaceEntry.Name().String())) if err != nil { - logger.Error("failed to construct forwarding request URL", tag.Operation("CompleteNexusOperation"), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.Error(err), tag.TargetCluster(targetCluster)) - return &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed"} + logger.Error("failed to construct forwarding request URL", tag.Error(err)) + return &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error"), Outcome: "request_forwarding_failed", SkipServiceErrorReporting: true} } - request.CompletionRequest.HTTPRequest.Header.Set(interceptor.DCRedirectionAPIHeaderName, "true") - request.CompletionRequest.HTTPRequest.Header.Set(interceptor.DCRedirectionSourceCellHeaderName, i.clusterMetadata.GetCurrentClusterName()) info.OriginalRequestHeaders.Set(interceptor.DCRedirectionAPIHeaderName, "true") info.OriginalRequestHeaders.Set(interceptor.DCRedirectionSourceCellHeaderName, i.clusterMetadata.GetCurrentClusterName()) completion, err := completeOperationOptions(request.CompletionRequest) if err != nil { - return err + return &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error", SkipServiceErrorReporting: true} } ctx = i.withForwardingTrace(ctx, "CompleteNexusOperation", "", "", info, namespaceEntry, targetCluster) err = nexusrpc.NewCompletionHTTPClient(nexusrpc.CompletionHTTPClientOptions{ @@ -221,7 +232,7 @@ func (i *nexusForwardingInterceptor) forwardCompleteOperation( HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: client, originalRequestHeaders: info.OriginalRequestHeaders}).Do, }).CompleteOperation(ctx, forwardURL, completion) if err != nil { - return &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error"} + return &interceptornexus.InterceptorError{Err: err, Outcome: "forwarded_request_error", SkipServiceErrorReporting: true} } return nil } @@ -239,6 +250,7 @@ func completeOperationOptions(request *nexusrpc.CompletionRequest) (nexusrpc.Com func (i *nexusForwardingInterceptor) nexusClientForActiveCluster( ctx context.Context, + logger log.Logger, service string, info interceptornexus.ForwardingInfo, namespaceEntry *namespace.Namespace, @@ -248,15 +260,10 @@ func (i *nexusForwardingInterceptor) nexusClientForActiveCluster( if oc, ok := operationContextFromContext(ctx); ok { setFailureSource = oc.setFailureSource } - logger := log.With( - i.logger, - tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), - tag.TargetCluster(targetCluster), - ) httpClient, err := i.forwardingClients.Get(targetCluster) if err != nil { - logger.Error("failed to forward Nexus request: error creating HTTP client", tag.Error(err), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) - return nil, &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} + logger.Error("failed to forward Nexus request: error creating HTTP client", tag.Error(err)) + return nil, &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed", SkipServiceErrorReporting: true} } var baseURL string if i.serviceConfig.NexusForwardRequestUseEndpoint() && info.EndpointID != "" { @@ -265,8 +272,8 @@ func (i *nexusForwardingInterceptor) nexusClientForActiveCluster( baseURL, err = url.JoinPath(httpClient.BaseURL(), commonnexus.RouteDispatchNexusTaskByNamespaceAndTaskQueue.Path(commonnexus.NamespaceAndTaskQueue{Namespace: namespaceEntry.Name().String(), TaskQueue: info.TaskQueue})) } if err != nil { - logger.Error("failed to forward Nexus request: error constructing ServiceBaseURL", tag.URL(httpClient.BaseURL()), tag.WorkflowNamespace(namespaceEntry.Name().String()), tag.WorkflowTaskQueueName(info.TaskQueue), tag.Error(err)) - return nil, &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed"} + logger.Error("failed to forward Nexus request: error constructing ServiceBaseURL", tag.URL(httpClient.BaseURL()), tag.WorkflowTaskQueueName(info.TaskQueue), tag.Error(err)) + return nil, &interceptornexus.InterceptorError{Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "request forwarding failed"), Outcome: "request_forwarding_failed", SkipServiceErrorReporting: true} } return nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ HTTPCaller: (&nexusForwardingHTTPHeaderWrapper{client: httpClient, originalRequestHeaders: info.OriginalRequestHeaders, setFailureSource: setFailureSource}).Do, @@ -287,16 +294,30 @@ func (i *nexusForwardingInterceptor) withForwardingTrace( if i.httpTraceProvider == nil { return ctx } - traceLogger := log.With(i.logger, - tag.Operation(method), - tag.WorkflowNamespace(namespaceEntry.Name().String()), - tag.RequestID(requestID), - tag.NexusOperation(operation), - tag.Endpoint(info.EndpointName), + traceLogger := i.logger + tags := []tag.Tag{ tag.AttemptStart(time.Now().UTC()), tag.SourceCluster(i.clusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster), - ) + } + if rCtx, ok := requestContextFromContext(ctx); ok { + traceLogger = rCtx.logger + } else { + tags = append(tags, + tag.Operation(method), + tag.WorkflowNamespace(namespaceEntry.Name().String()), + ) + if requestID != "" { + tags = append(tags, tag.RequestID(requestID)) + } + if operation != "" { + tags = append(tags, tag.NexusOperation(operation)) + } + if info.EndpointName != "" { + tags = append(tags, tag.Endpoint(info.EndpointName)) + } + } + traceLogger = log.With(traceLogger, tags...) if trace := i.httpTraceProvider.NewForwardingTrace(traceLogger); trace != nil { return httptrace.WithClientTrace(ctx, trace) } diff --git a/service/frontend/nexus_forward_interceptor_test.go b/service/frontend/nexus_forward_interceptor_test.go index fc116ef36fe..e3caef35b03 100644 --- a/service/frontend/nexus_forward_interceptor_test.go +++ b/service/frontend/nexus_forward_interceptor_test.go @@ -8,7 +8,9 @@ import ( "io" "net/http" "net/http/httptest" + "strconv" "testing" + "time" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" @@ -38,8 +40,10 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { requestForwarded ) - // dummy server to simulate fowarded req + var receivedHeaders http.Header + // dummy server to simulate forwarded req server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + receivedHeaders = request.Header.Clone() w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusCreated) _, _ = fmt.Fprint(w, `{"token":"operation-token","state":"running"}`) @@ -52,22 +56,11 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { Scheme: "http", }, }} - options := nexus.StartOperationOptions{ - Header: nexus.Header{"X-Request": "request"}, - } - requestInput := nexus.NewLazyValue(nexus.DefaultSerializer(), &nexus.Reader{ - ReadCloser: io.NopCloser(bytes.NewBufferString(`"input"`)), - Header: nexus.Header{"type": "json"}, - }) - forwardingInfo := interceptornexus.ForwardingInfo{ - OriginalRequestHeaders: http.Header{"X-Original": {"original"}}, - TaskQueue: "task-queue", - } - for _, tc := range []struct { name string namespace *namespace.Namespace forwardingOn bool + redirectAllowed *bool expectedOutcome string disposition requestDisposition }{ @@ -105,6 +98,20 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { expectedOutcome: "namespace_inactive_forwarding_disabled", disposition: requestFailed, }, + { + name: "global namespace with redirection disabled should fail", + namespace: namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace}, + nil, + true, + &persistencespb.NamespaceReplicationConfig{ActiveClusterName: remoteCluster, Clusters: []string{currentCluster, remoteCluster}}, + 0, + ), + forwardingOn: true, + redirectAllowed: new(false), + expectedOutcome: "namespace_inactive_forwarding_disabled", + disposition: requestFailed, + }, { name: "global namespace with forwarding enabled to unknown cluster fails", namespace: namespace.NewNamespaceForTest( @@ -120,6 +127,19 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + receivedHeaders = nil + options := nexus.StartOperationOptions{Header: nexus.Header{"X-Request": "request"}} + if tc.redirectAllowed != nil { + options.Header[interceptor.DCRedirectionContextHeaderName] = strconv.FormatBool(*tc.redirectAllowed) + } + requestInput := nexus.NewLazyValue(nexus.DefaultSerializer(), &nexus.Reader{ + ReadCloser: io.NopCloser(bytes.NewBufferString(`"input"`)), + Header: nexus.Header{"type": "json"}, + }) + forwardingInfo := interceptornexus.ForwardingInfo{ + OriginalRequestHeaders: http.Header{"X-Original": {"original"}}, + TaskQueue: "task-queue", + } forwarder := &nexusForwardingInterceptor{ logger: log.NewNoopLogger(), clusterMetadata: metadata, @@ -141,7 +161,7 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { }, } in := interceptornexus.NewStartOpInput( - "s", "o", testNamespace, options, requestInput, + "s", "o", testNamespace, time.Now(), options, requestInput, forwardingInfo, interceptornexus.RequestMetadata{NamespaceEntry: tc.namespace}, ) @@ -159,6 +179,7 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { var interceptorErr *interceptornexus.InterceptorError require.ErrorAs(t, err, &interceptorErr) require.Equal(t, tc.expectedOutcome, interceptorErr.Outcome) + require.True(t, interceptorErr.SkipServiceErrorReporting) } else { require.NoError(t, err) } @@ -169,6 +190,8 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { require.Equal(t, requestHandledLocally, result) case requestForwarded: require.IsType(t, &nexus.HandlerStartOperationResultAsync{}, result) + require.Equal(t, "true", receivedHeaders.Get(interceptor.DCRedirectionAPIHeaderName)) + require.Equal(t, currentCluster, receivedHeaders.Get(interceptor.DCRedirectionSourceCellHeaderName)) case requestFailed: require.Nil(t, result) default: diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index 588d2be718e..bc9a04c5a16 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -61,17 +61,13 @@ type nexusContext struct { // Context for a specific Nexus operation, includes a resolved namespace, and a bound metrics handler and logger. type operationContext struct { *nexusContext - method string - clusterMetadata cluster.Metadata - namespace *namespace.Namespace + method string + namespace *namespace.Namespace // "Special" metrics handler that should only be passed to interceptors, which require a different set of // pre-baked tags than the "normal" metricsHandler. metricsHandlerForInterceptors metrics.Handler - metricsHandler metrics.Handler logger log.Logger - clientVersionChecker headers.VersionChecker requestErrorHandler *interceptor.RequestErrorHandler - headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] } func (c *operationContext) matchingRequest(req *nexuspb.Request) *matchingservice.DispatchNexusTaskRequest { @@ -125,8 +121,6 @@ func (c *operationContext) handleRequestError(err error) { return } c.requestErrorHandler.HandleError( - // The request is only read to extract workflow log tags, which is keyed off the - // gRPC full method. Nexus has none, so it is never used. nil, "", c.metricsHandlerForInterceptors, @@ -136,20 +130,6 @@ func (c *operationContext) handleRequestError(err error) { ) } -// required as operations might panic before the interceptor chain is invoked -func captureOperationPanic(logger log.Logger, errPtr *error) { - recovered := recover() //nolint:revive - if recovered == nil { - return - } - err, ok := recovered.(error) - if !ok { - err = fmt.Errorf("panic: %v", recovered) - } - logger.Error("Panic captured", tag.SysStackTrace(string(debug.Stack())), tag.Error(err)) - *errPtr = err -} - // convertInterceptorError converts the error returned by the interceptor chain into the sanitized // form returned to the Nexus caller, hiding internal error detail. Interceptors intentionally leave // InterceptorError.Err raw so the boundary can log/classify the full original error via @@ -158,18 +138,26 @@ func convertInterceptorError(err error) error { if err == nil { return nil } + exposeDetails := false if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { - // always convert error to omit exposing details to end callers - return commonnexus.ConvertGRPCError(taggedErr.Err, false) + err = taggedErr.Err + exposeDetails = taggedErr.ExposeDetails } - return err + return commonnexus.ConvertGRPCError(err, exposeDetails) } // finalizeOperationRequest is the single deferred step for a Nexus start/cancel operation: capture // a panic into errPtr, log/classify the (still raw) resulting error, then sanitize it for the // response. Order matters and must not be split back into separate defers. func finalizeOperationRequest(oc *operationContext, errPtr *error) { - captureOperationPanic(oc.logger, errPtr) + if recovered := recover(); recovered != nil { //nolint:revive + err, ok := recovered.(error) + if !ok { + err = fmt.Errorf("panic: %v", recovered) + } + oc.logger.Error("Panic captured", tag.SysStackTrace(string(debug.Stack())), tag.Error(err)) + *errPtr = err + } oc.handleRequestError(*errPtr) *errPtr = convertInterceptorError(*errPtr) } @@ -223,18 +211,16 @@ func operationContextFromContext(ctx context.Context) (*operationContext, bool) // Dispatches Nexus requests as Nexus tasks to workers via matching. type nexusHandler struct { nexus.UnimplementedHandler - logger log.Logger - metricsHandler metrics.Handler - clusterMetadata cluster.Metadata - namespaceRegistry namespace.Registry - matchingClient matchingservice.MatchingServiceClient - requestErrorHandler *interceptor.RequestErrorHandler - payloadSizeLimit dynamicconfig.IntPropertyFnWithNamespaceFilter - headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] - useForwardByEndpoint dynamicconfig.BoolPropertyFn - metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig] - httpTraceProvider commonnexus.HTTPClientTraceProvider - chainedHandler interceptornexus.HandlerFunc + logger log.Logger + metricsHandler metrics.Handler + clusterMetadata cluster.Metadata + namespaceRegistry namespace.Registry + matchingClient matchingservice.MatchingServiceClient + requestErrorHandler *interceptor.RequestErrorHandler + payloadSizeLimit dynamicconfig.IntPropertyFnWithNamespaceFilter + headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] + metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig] + chainedHandler interceptornexus.HandlerFunc } func newNexusHandler( @@ -246,23 +232,19 @@ func newNexusHandler( requestErrorHandler *interceptor.RequestErrorHandler, payloadSizeLimit dynamicconfig.IntPropertyFnWithNamespaceFilter, headersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp], - useForwardByEndpoint dynamicconfig.BoolPropertyFn, metricTagConfig dynamicconfig.TypedPropertyFn[chasmnexus.NexusMetricTagConfig], - httpTraceProvider commonnexus.HTTPClientTraceProvider, nexusInterceptors []interceptornexus.Interceptor, ) *nexusHandler { h := &nexusHandler{ - logger: logger, - metricsHandler: metricsHandler, - clusterMetadata: clusterMetadata, - namespaceRegistry: namespaceRegistry, - matchingClient: matchingClient, - requestErrorHandler: requestErrorHandler, - payloadSizeLimit: payloadSizeLimit, - headersBlacklist: headersBlacklist, - useForwardByEndpoint: useForwardByEndpoint, - metricTagConfig: metricTagConfig, - httpTraceProvider: httpTraceProvider, + logger: logger, + metricsHandler: metricsHandler, + clusterMetadata: clusterMetadata, + namespaceRegistry: namespaceRegistry, + matchingClient: matchingClient, + requestErrorHandler: requestErrorHandler, + payloadSizeLimit: payloadSizeLimit, + headersBlacklist: headersBlacklist, + metricTagConfig: metricTagConfig, } h.chainedHandler = interceptornexus.ChainInterceptors(h.finalHandler, nexusInterceptors) return h @@ -294,12 +276,9 @@ func (h *nexusHandler) getOperationContext(ctx context.Context, method string) ( return nil, errors.New("no nexus context set on context") } oc := operationContext{ - nexusContext: nc, - method: method, - clusterMetadata: h.clusterMetadata, - clientVersionChecker: headers.NewDefaultVersionChecker(), - requestErrorHandler: h.requestErrorHandler, - headersBlacklist: h.headersBlacklist, + nexusContext: nc, + method: method, + requestErrorHandler: h.requestErrorHandler, } oc.metricsHandlerForInterceptors = h.metricsHandler.WithTags( metrics.OperationTag(method), @@ -308,7 +287,7 @@ func (h *nexusHandler) getOperationContext(ctx context.Context, method string) ( var err error if oc.namespace, err = h.namespaceRegistry.GetNamespace(namespace.Name(nc.namespaceName)); err != nil { - // draft-review: should this block be removed now that this is in an interceptor? + // namespace lookup runs before the interceptor chain, so this outcome is recorded here. metrics.NexusRequests.With(h.metricsHandler).Record( 1, metrics.NamespaceTag(nc.namespaceName), @@ -344,11 +323,36 @@ func (h *nexusHandler) StartOperation( defer finalizeOperationRequest(oc, &retErr) ctx = withOperationContext(ctx, oc) + var links []*nexuspb.Link + for _, nexusLink := range options.Links { + links = append(links, &nexuspb.Link{ + Url: nexusLink.URL.String(), + Type: nexusLink.Type, + }) + } + request := oc.matchingRequest(&nexuspb.Request{ + ScheduledTime: timestamppb.New(oc.requestStartTime), + Header: options.Header, + Variant: &nexuspb.Request_StartOperation{ + StartOperation: &nexuspb.StartOperationRequest{ + Service: service, + Operation: operation, + Callback: options.CallbackURL, + CallbackHeader: options.CallbackHeader, + RequestId: options.RequestID, + Links: links, + }, + }, + Capabilities: &nexuspb.Request_Capabilities{ + TemporalFailureResponses: oc.callerFailureSupport, + }, + }) nexusOpInput := interceptornexus.NewStartOpInput( service, operation, oc.namespaceName, + oc.requestStartTime, options, input, interceptornexus.ForwardingInfo{ @@ -362,6 +366,7 @@ func (h *nexusHandler) StartOperation( NamespaceEntry: oc.namespace, EndpointName: oc.endpointName, MetricTags: h.nexusMetricTags(service, operation, options.Header), + Request: request, }, ) out, err := h.chainedHandler(ctx, nexusOpInput) @@ -385,43 +390,19 @@ func (h *nexusHandler) finalStartHandler( return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid operation context for nexus start operation") } operation := in.OperationName() - var input *nexus.LazyValue - var options nexus.StartOperationOptions - if soi, ok := in.(interceptornexus.StartOpInput); !ok { + soi, ok := in.(interceptornexus.StartOpInput) + if !ok { return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid request for nexus start operation") - } else { - input = soi.StartOperationInput - options = soi.StartOperationOptions } - var links []*nexuspb.Link - for _, nexusLink := range options.Links { - links = append(links, &nexuspb.Link{ - Url: nexusLink.URL.String(), - Type: nexusLink.Type, - }) + request, ok := soi.Request().(*matchingservice.DispatchNexusTaskRequest) + if !ok || request.GetRequest().GetStartOperation() == nil { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid dispatch request for nexus start operation") } - startOperationRequest := &nexuspb.StartOperationRequest{ - Service: in.ServiceName(), - Operation: operation, - Callback: options.CallbackURL, - CallbackHeader: options.CallbackHeader, - RequestId: options.RequestID, - Links: links, - } - request := oc.matchingRequest(&nexuspb.Request{ - ScheduledTime: timestamppb.New(oc.requestStartTime), - Header: options.Header, - Variant: &nexuspb.Request_StartOperation{ - StartOperation: startOperationRequest, - }, - Capabilities: &nexuspb.Request_Capabilities{ - TemporalFailureResponses: oc.callerFailureSupport, - }, - }) + startOperationRequest := request.GetRequest().GetStartOperation() h.sanitizeRequestHeaders(request) var err error // Transform nexus Content to temporal Payload with common/nexus PayloadSerializer. - if err = input.Consume(&startOperationRequest.Payload); err != nil { + if err = soi.StartOperationInput.Consume(&startOperationRequest.Payload); err != nil { oc.logger.Warn("invalid input", tag.Error(err)) return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid input") } @@ -478,11 +459,28 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, oc.annotateServerSpan(ctx, service, operation, "") // for edge case where the operation panics before the interceptor chain is invoked defer finalizeOperationRequest(oc, &retErr) + request := oc.matchingRequest(&nexuspb.Request{ + Header: options.Header, + ScheduledTime: timestamppb.New(oc.requestStartTime), + Variant: &nexuspb.Request_CancelOperation{ + CancelOperation: &nexuspb.CancelOperationRequest{ + Service: service, + Operation: operation, + OperationToken: token, + // TODO(bergundy): Remove this fallback after the 1.27 release. + OperationId: token, + }, + }, + Capabilities: &nexuspb.Request_Capabilities{ + TemporalFailureResponses: oc.callerFailureSupport, + }, + }) nexusInterceptorInput := interceptornexus.NewCancelOpInput( service, operation, oc.namespaceName, + oc.requestStartTime, options, token, interceptornexus.ForwardingInfo{ @@ -496,6 +494,7 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, NamespaceEntry: oc.namespace, EndpointName: oc.endpointName, MetricTags: h.nexusMetricTags(service, operation, options.Header), + Request: request, }, ) ctx = withOperationContext(ctx, oc) @@ -529,26 +528,11 @@ func (h *nexusHandler) finalCancelHandler( if !ok { return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid request for nexus cancel operation") } - options := coi.CancelOperationOptions - token := coi.CancellationToken - operation := in.OperationName() - request := oc.matchingRequest(&nexuspb.Request{ - Header: options.Header, - ScheduledTime: timestamppb.New(oc.requestStartTime), - Variant: &nexuspb.Request_CancelOperation{ - CancelOperation: &nexuspb.CancelOperationRequest{ - Service: in.ServiceName(), - Operation: operation, - OperationToken: token, - // TODO(bergundy): Remove this fallback after the 1.27 release. - can this be removed now? - OperationId: token, - }, - }, - Capabilities: &nexuspb.Request_Capabilities{ - TemporalFailureResponses: oc.callerFailureSupport, - }, - }) + request, ok := coi.Request().(*matchingservice.DispatchNexusTaskRequest) + if !ok || request.GetRequest().GetCancelOperation() == nil { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "invalid dispatch request for nexus cancel operation") + } h.sanitizeRequestHeaders(request) // Dispatch the request to be sync matched with a worker polling on the nexusContext taskQueue. @@ -566,25 +550,6 @@ func (h *nexusHandler) finalCancelHandler( return nil, oc.handleCancelOperationResponse(response, operation) } -func convertOutcomeToNexusHandlerError(resp *matchingservice.DispatchNexusTaskResponse_HandlerError) *nexus.HandlerError { - var retryBehavior nexus.HandlerErrorRetryBehavior - // nolint:exhaustive // unspecified is the default - switch resp.HandlerError.RetryBehavior { - case enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE: - retryBehavior = nexus.HandlerErrorRetryBehaviorRetryable - case enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_NON_RETRYABLE: - retryBehavior = nexus.HandlerErrorRetryBehaviorNonRetryable - } - // nolint:staticcheck // Deprecated function still in use for backward compatibility. - cause := commonnexus.ProtoFailureToNexusFailure(resp.HandlerError.GetFailure()) - return &nexus.HandlerError{ - // nolint:staticcheck // Deprecated function still in use for backward compatibility. - Type: nexus.HandlerErrorType(resp.HandlerError.GetErrorType()), - RetryBehavior: retryBehavior, - Cause: &nexus.FailureError{Failure: cause}, - } -} - func (nc *nexusContext) setFailureSource(source string) { nc.responseHeadersMutex.Lock() defer nc.responseHeadersMutex.Unlock() diff --git a/service/frontend/nexus_handler_test.go b/service/frontend/nexus_handler_test.go index 81138006b53..6d28ca3c533 100644 --- a/service/frontend/nexus_handler_test.go +++ b/service/frontend/nexus_handler_test.go @@ -1,101 +1,38 @@ package frontend import ( - "context" - "errors" - "time" - "github.com/google/uuid" enumspb "go.temporal.io/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" - "go.temporal.io/server/common/authorization" "go.temporal.io/server/common/cluster" - "go.temporal.io/server/common/cluster/clustertest" - "go.temporal.io/server/common/dynamicconfig" - "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/primitives/timestamp" - "go.temporal.io/server/common/quotas" ) -type mockAuthorizer struct{} - -// Authorize implements authorization.Authorizer. -func (mockAuthorizer) Authorize(ctx context.Context, caller *authorization.Claims, target *authorization.CallTarget) (authorization.Result, error) { - return authorization.Result{Decision: authorization.DecisionAllow}, nil -} - -var _ authorization.Authorizer = mockAuthorizer{} - -type mockRateLimiter struct { - allow bool -} - -// Allow implements quotas.RequestRateLimiter. -func (r mockRateLimiter) Allow(now time.Time, request quotas.Request) bool { - return r.allow -} - -// Reserve implements quotas.RequestRateLimiter. -func (mockRateLimiter) Reserve(now time.Time, request quotas.Request) quotas.Reservation { - panic("unimplemented for test") -} - -// Wait implements quotas.RequestRateLimiter. -func (mockRateLimiter) Wait(ctx context.Context, request quotas.Request) error { - panic("unimplemented for test") -} - -var _ quotas.RequestRateLimiter = mockRateLimiter{} - -type mockNamespaceChecker namespace.Name - -func (n mockNamespaceChecker) Exists(name namespace.Name) error { - if name == namespace.Name(n) { - return nil - } - return errors.New("doesn't exist") -} - -type contextOptions struct { - namespaceState enumspb.NamespaceState - namespacePassive bool - quota int - namespaceRateLimitAllow bool - rateLimitAllow bool - redirectAllow bool - headersBlacklist []string -} - -func newOperationContext(options contextOptions) *operationContext { +func newOperationContext() *operationContext { oc := &operationContext{ nexusContext: &nexusContext{}, } oc.logger = log.NewTestLogger() oc.metricsHandlerForInterceptors = metricstest.NewCaptureHandler() - oc.clientVersionChecker = headers.NewDefaultVersionChecker() oc.apiName = "/temporal.api.nexusservice.v1.NexusService/DispatchNexusTask" oc.responseHeaders = make(map[string]string) oc.namespaceName = "test-namespace" - activeClusterName := cluster.TestCurrentClusterName - if options.namespacePassive { - activeClusterName = cluster.TestAlternativeClusterName - } oc.namespace = namespace.NewGlobalNamespaceForTest( &persistencespb.NamespaceInfo{ Id: uuid.NewString(), Name: oc.namespaceName, - State: options.namespaceState, + State: enumspb.NAMESPACE_STATE_REGISTERED, }, &persistencespb.NamespaceConfig{ Retention: timestamp.DurationFromDays(1), CustomSearchAttributeAliases: make(map[string]string), }, &persistencespb.NamespaceReplicationConfig{ - ActiveClusterName: activeClusterName, + ActiveClusterName: cluster.TestCurrentClusterName, Clusters: []string{ cluster.TestCurrentClusterName, cluster.TestAlternativeClusterName, @@ -104,14 +41,5 @@ func newOperationContext(options contextOptions) *operationContext { 1, ) - oc.clusterMetadata = clustertest.NewMetadataForTest( - cluster.NewTestClusterMetadataConfig(true, !options.namespacePassive), - ) - re, err := dynamicconfig.ConvertWildcardStringListToRegexp(options.headersBlacklist) - if err != nil { - panic(err) // nolint:forbidigo - } - oc.headersBlacklist = dynamicconfig.GetTypedPropertyFn(re) - return oc } diff --git a/service/frontend/nexus_interceptor_chain_test.go b/service/frontend/nexus_interceptor_chain_test.go new file mode 100644 index 00000000000..4050757bbce --- /dev/null +++ b/service/frontend/nexus_interceptor_chain_test.go @@ -0,0 +1,175 @@ +package frontend + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestNexusChainPreservesNativeErrors(t *testing.T) { + tests := []struct { + name string + err error + outcome string + wrapError bool + assertErrors func(*testing.T, error, bool) + }{ + { + name: "operation error", + err: &nexus.OperationError{ + Message: "operation failed", + State: nexus.OperationStateFailed, + Cause: errors.New("worker failure"), + }, + outcome: "operation_error", + wrapError: true, + assertErrors: func(t *testing.T, err error, _ bool) { + var operationErr *nexus.OperationError + require.ErrorAs(t, err, &operationErr) + require.Equal(t, "operation failed", operationErr.Message) + + convertedErr := convertInterceptorError(err) + require.ErrorAs(t, convertedErr, &operationErr) + require.Equal(t, nexus.OperationStateFailed, operationErr.State) + require.Equal(t, "operation failed", operationErr.Message) + }, + }, + { + name: "handler error", + err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid input"), + outcome: "handler_error", + wrapError: true, + assertErrors: func(t *testing.T, err error, _ bool) { + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeBadRequest, handlerErr.Type) + require.Equal(t, "invalid input", handlerErr.Message) + + require.ErrorAs(t, convertInterceptorError(err), &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeBadRequest, handlerErr.Type) + }, + }, + { + name: "bare handler error", + err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid input"), + outcome: "internal_error", + wrapError: false, + assertErrors: func(t *testing.T, err error, _ bool) { + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeBadRequest, handlerErr.Type) + require.Equal(t, "invalid input", handlerErr.Message) + }, + }, + { + name: "internal gRPC error", + err: status.Error(codes.Internal, "worker failure"), + outcome: "internal_error", + wrapError: true, + assertErrors: func(t *testing.T, err error, maskErrors bool) { + require.Equal(t, codes.Internal, status.Code(err)) + if maskErrors { + require.NotContains(t, err.Error(), "worker failure") + } else { + require.ErrorContains(t, err, "worker failure") + } + + var handlerErr *nexus.HandlerError + require.ErrorAs(t, convertInterceptorError(err), &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type) + require.Equal(t, "internal error", handlerErr.Message) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + for _, maskErrors := range []bool{false, true} { + t.Run(fmt.Sprintf("mask errors=%t", maskErrors), func(t *testing.T) { + t.Parallel() + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + + chainedHandler := newTestNexusInterceptorChain(metricsHandler, maskErrors, tc.err, tc.outcome, tc.wrapError) + _, err := chainedHandler(context.Background(), newTestNexusStartInput()) + + rawErr := err + if tc.wrapError { + var interceptorErr *interceptornexus.InterceptorError + require.ErrorAs(t, err, &interceptorErr) + require.Equal(t, tc.outcome, interceptorErr.Outcome) + rawErr = interceptorErr.Err + } + tc.assertErrors(t, rawErr, maskErrors) + + snapshot := capture.Snapshot() + require.Len(t, snapshot[metrics.NexusRequests.Name()], 1) + require.Equal(t, tc.outcome, snapshot[metrics.NexusRequests.Name()][0].Tags["outcome"]) + }) + } + }) + } +} + +func newTestNexusInterceptorChain( + metricsHandler metrics.Handler, + maskErrors bool, + terminalErr error, + outcome string, + wrapError bool, +) interceptornexus.HandlerFunc { + telemetry := rpcinterceptor.NewTelemetryInterceptor(nil, metricsHandler, log.NewNoopLogger(), nil, nil) + mask := rpcinterceptor.NewMaskInternalErrorDetailsInterceptor( + dynamicconfig.GetBoolPropertyFnFilteredByNamespace(maskErrors), + nil, + log.NewNoopLogger(), + ) + serviceErrors := rpcinterceptor.NewServiceErrorInterceptor( + dynamicconfig.GetIntPropertyFn(4000), + metrics.NoopMetricsHandler, + log.NewNoopLogger(), + ) + frontendServiceErrors := rpcinterceptor.NewFrontendServiceErrorInterceptorWrapper(log.NewNoopLogger()) + + return interceptornexus.ChainInterceptors( + func(context.Context, interceptornexus.InterceptorInput) (any, error) { + if !wrapError { + return nil, terminalErr + } + return nil, &interceptornexus.InterceptorError{Err: terminalErr, Outcome: outcome} + }, + []interceptornexus.Interceptor{ + telemetry.InterceptNexusOutermost, + mask.InterceptNexus, + serviceErrors.InterceptNexus, + frontendServiceErrors.InterceptNexus, + }, + ) +} + +func newTestNexusStartInput() interceptornexus.StartOpInput { + return interceptornexus.NewStartOpInput( + "s", + "o", + testNamespace, + time.Now(), + nexus.StartOperationOptions{}, + nil, + interceptornexus.ForwardingInfo{}, + interceptornexus.RequestMetadata{NamespaceEntry: testOperationContext().namespace}, + ) +} diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index ccb0f3bbe62..fb3fa3e5302 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -52,7 +52,6 @@ func NewNexusOperationHTTPHandler( matchingClient resource.MatchingClient, metricsHandler metrics.Handler, clusterMetadata cluster.Metadata, - clientCache *cluster.FrontendHTTPClientCache, namespaceRegistry namespace.Registry, endpointRegistry commonnexus.EndpointRegistry, authInterceptor *authorization.Interceptor, @@ -60,7 +59,6 @@ func NewNexusOperationHTTPHandler( requestErrorHandler *interceptor.RequestErrorHandler, interceptorsProvider *InterceptorsProvider, logger log.Logger, - httpTraceProvider commonnexus.HTTPClientTraceProvider, httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter, ) *NexusOperationHTTPHandler { logger = log.With(logger, tag.NexusStageHandlerInbound) @@ -87,9 +85,7 @@ func NewNexusOperationHTTPHandler( requestErrorHandler, serviceConfig.BlobSizeLimitError, serviceConfig.NexusRequestHeadersBlacklist, - serviceConfig.NexusForwardRequestUseEndpoint, serviceConfig.NexusOperationsMetricTagConfig, - httpTraceProvider, interceptorsProvider.NexusInterceptors(), ), GetResultTimeout: serviceConfig.KeepAliveMaxConnectionIdle(), @@ -150,7 +146,7 @@ func (h *NexusOperationHTTPHandler) dispatchNexusTaskByNamespaceAndTaskQueue(w h return } - rWithAuthCtx, err := h.parseTLSAndAuthInfo(r, nc) + rWithAuthCtx, err := h.parseTLSAndAuthInfo(r) if err != nil { logger.Error("failed to get claims", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnauthenticated, "unauthorized")) @@ -213,7 +209,7 @@ func (h *NexusOperationHTTPHandler) dispatchNexusTaskByEndpoint(w http.ResponseW return } - rWithAuthCtx, err := h.parseTLSAndAuthInfo(r, nc) + rWithAuthCtx, err := h.parseTLSAndAuthInfo(r) if err != nil { logger.Error("failed to get claims", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnauthenticated, "unauthorized")) @@ -293,7 +289,7 @@ func prepareRequest[T any](route routing.Route[T], w http.ResponseWriter, r *htt return route.Deserialize(vars) } -func (h *NexusOperationHTTPHandler) parseTLSAndAuthInfo(r *http.Request, nc *nexusContext) (*http.Request, error) { +func (h *NexusOperationHTTPHandler) parseTLSAndAuthInfo(r *http.Request) (*http.Request, error) { var tlsInfo *credentials.TLSInfo if r.TLS != nil { tlsInfo = &credentials.TLSInfo{ diff --git a/temporal/server_option.go b/temporal/server_option.go index 04498b9e61c..390e56fc7f3 100644 --- a/temporal/server_option.go +++ b/temporal/server_option.go @@ -202,6 +202,8 @@ func WithSearchAttributesMapper(m searchattribute.Mapper) ServerOption { // Frontend gRPC API calls. The list of custom interceptors will be appended to the end of the internal // ServerInterceptors. The custom interceptors will be invoked in the order as they appear in the supplied list, after // the internal ServerInterceptors. +// +// Deprecated: Use WithChainedFrontendInterceptors instead. func WithChainedFrontendGrpcInterceptors( interceptors ...grpc.UnaryServerInterceptor, ) ServerOption { @@ -211,9 +213,8 @@ func WithChainedFrontendGrpcInterceptors( } // WithChainedFrontendInterceptors sets an ordered chain of custom gRPC+Nexus interceptors that will be invoked for all -// Frontend gRPC and Nexus API calls respectively. The list of custom interceptors will be appended to the end of the internal -// ServerInterceptors. The custom interceptors will be invoked in the order as they appear in the supplied list, after -// the internal ServerInterceptors. +// Frontend gRPC and Nexus API calls respectively. Custom interceptors run after the internal +// interceptors and before the fault-injection and retryable interceptors, in the order supplied. func WithChainedFrontendInterceptors( interceptors ...frontend.Interceptor, ) ServerOption { diff --git a/temporal/server_options.go b/temporal/server_options.go index 123a0c19e6d..bb192d79bb3 100644 --- a/temporal/server_options.go +++ b/temporal/server_options.go @@ -132,6 +132,12 @@ func (so *serverOptions) loadConfig() error { } func (so *serverOptions) validateConfig() error { + if len(so.customFrontendInterceptors) > 0 && + len(so.customFrontendUnifiedInterceptors) > 0 { + // Both could be supported as a migration path but intentionally avoided as + // migration itself is as simple as wrapping with no-op Nexus Interceptors. + return errors.New("configure either custom gRPC or unified interceptors, not both") + } if err := so.config.Validate(); err != nil { return err } From 113f9c9907cf49f52a798c9571cc05dcaf455dd9 Mon Sep 17 00:00:00 2001 From: Maruthi ChandraSekhar Vemuri Date: Mon, 7 Sep 2026 21:33:22 -0700 Subject: [PATCH 12/12] address claude feedback --- chasm/interceptors.go | 6 +- common/authorization/interceptor.go | 2 +- .../rpc/interceptor/frontend_service_error.go | 2 + common/rpc/interceptor/mask_internal_error.go | 16 +-- common/rpc/interceptor/namespace_handover.go | 43 +------ common/rpc/interceptor/namespace_validator.go | 21 ++-- .../interceptor/namespace_validator_test.go | 2 +- common/rpc/interceptor/nexus/nexus.go | 6 +- common/rpc/interceptor/nexus/nexus_test.go | 69 +++++++++++ .../interceptor/service_error_interceptor.go | 43 +++++-- common/rpc/interceptor/telemetry.go | 2 +- service/frontend/frontend_interceptors.go | 16 +-- service/frontend/fx.go | 14 +-- .../frontend/nexus_completion_http_handler.go | 47 ++++--- service/frontend/nexus_dispatch_result.go | 8 +- .../frontend/nexus_dispatch_result_test.go | 6 +- service/frontend/nexus_forward_interceptor.go | 14 ++- .../nexus_forward_interceptor_test.go | 1 + service/frontend/nexus_handler.go | 7 +- service/frontend/nexus_handler_test.go | 2 +- .../frontend/nexus_interceptor_chain_test.go | 117 ++++++++++++++++-- .../frontend/nexus_operation_http_handler.go | 7 +- temporal/server_option.go | 3 +- temporal/server_options.go | 2 +- temporal/server_options_test.go | 46 +++++++ 25 files changed, 358 insertions(+), 144 deletions(-) create mode 100644 temporal/server_options_test.go diff --git a/chasm/interceptors.go b/chasm/interceptors.go index 578879fa5f7..06a58c52de2 100644 --- a/chasm/interceptors.go +++ b/chasm/interceptors.go @@ -5,7 +5,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" - n "go.temporal.io/server/common/rpc/interceptor/nexus" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "google.golang.org/grpc" ) @@ -63,8 +63,8 @@ func (i *ChasmVisibilityInterceptor) Intercept( func (i *ChasmVisibilityInterceptor) InterceptNexus( ctx context.Context, - in n.InterceptorInput, - next n.HandlerFunc, + in interceptornexus.InterceptorInput, + next interceptornexus.HandlerFunc, ) (any, error) { ctx = NewVisibilityManagerContext(ctx, i.visibilityMgr) return next(ctx, in) diff --git a/common/authorization/interceptor.go b/common/authorization/interceptor.go index 5bc0bb1796d..a2d41746266 100644 --- a/common/authorization/interceptor.go +++ b/common/authorization/interceptor.go @@ -191,7 +191,7 @@ func (a *Interceptor) InterceptNexus( } } logTags := []tag.Tag{ - tag.Operation(apiName), + tag.Operation(api.MethodName(apiName)), tag.WorkflowNamespace(namespaceName), tag.Endpoint(endpointName), tag.Error(err), diff --git a/common/rpc/interceptor/frontend_service_error.go b/common/rpc/interceptor/frontend_service_error.go index ed6ba1ea9a2..92720491ec7 100644 --- a/common/rpc/interceptor/frontend_service_error.go +++ b/common/rpc/interceptor/frontend_service_error.go @@ -36,6 +36,8 @@ func NewFrontendServiceErrorInterceptorWrapper(logger log.Logger) *FrontendServi } // NewFrontendServiceErrorInterceptor provides the legacy standalone gRPC Interceptor for existing deployments. +// +// Deprecated: use the unified [NewFrontendServiceErrorInterceptorWrapper] instead. func NewFrontendServiceErrorInterceptor(logger log.Logger) grpc.UnaryServerInterceptor { t := NewFrontendServiceErrorInterceptorWrapper(logger) return t.Intercept diff --git a/common/rpc/interceptor/mask_internal_error.go b/common/rpc/interceptor/mask_internal_error.go index 1efb495e33f..62bdf02b313 100644 --- a/common/rpc/interceptor/mask_internal_error.go +++ b/common/rpc/interceptor/mask_internal_error.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - nexusrpc "github.com/nexus-rpc/sdk-go/nexus" + "github.com/nexus-rpc/sdk-go/nexus" "go.temporal.io/api/serviceerror" "go.temporal.io/server/common" "go.temporal.io/server/common/api" @@ -14,7 +14,7 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/rpc/interceptor/logtags" - "go.temporal.io/server/common/rpc/interceptor/nexus" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/tasktoken" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -61,8 +61,8 @@ func (mi *MaskInternalErrorDetailsInterceptor) Intercept( func (mi *MaskInternalErrorDetailsInterceptor) InterceptNexus( ctx context.Context, - in nexus.InterceptorInput, - next nexus.HandlerFunc, + in interceptornexus.InterceptorInput, + next interceptornexus.HandlerFunc, ) (any, error) { resp, err := next(ctx, in) @@ -70,7 +70,7 @@ func (mi *MaskInternalErrorDetailsInterceptor) InterceptNexus( if err == nil || !mi.shouldMaskErrors(in) { return resp, err } - if ie, ok := errors.AsType[*nexus.InterceptorError](err); ok { + if ie, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { ie.Err = mi.maskNexusError(in, ie.Err) err = ie } else { @@ -87,11 +87,11 @@ func (mi *MaskInternalErrorDetailsInterceptor) shouldMaskErrors(req any) bool { return mi.maskInternalError(ns.String()) } -func (mi *MaskInternalErrorDetailsInterceptor) maskNexusError(in nexus.InterceptorInput, err error) error { - if _, ok := errors.AsType[*nexusrpc.HandlerError](err); ok { +func (mi *MaskInternalErrorDetailsInterceptor) maskNexusError(in interceptornexus.InterceptorInput, err error) error { + if _, ok := errors.AsType[*nexus.HandlerError](err); ok { return err } - if _, ok := errors.AsType[*nexusrpc.OperationError](err); ok { + if _, ok := errors.AsType[*nexus.OperationError](err); ok { return err } if _, ok := common.GetRPCStatus(err); !ok { diff --git a/common/rpc/interceptor/namespace_handover.go b/common/rpc/interceptor/namespace_handover.go index 69822d8d251..6328f2554f2 100644 --- a/common/rpc/interceptor/namespace_handover.go +++ b/common/rpc/interceptor/namespace_handover.go @@ -92,50 +92,13 @@ func (i *NamespaceHandoverInterceptor) handlesMethod(fullMethod string) bool { return false } +// InterceptNexus is a no-op: the handover gate only applies to WorkflowService +// methods- see [NamespaceHandoverInterceptor.handlesMethod] for details. func (i *NamespaceHandoverInterceptor) InterceptNexus( ctx context.Context, in nexus.InterceptorInput, next nexus.HandlerFunc, -) (_ any, retError error) { - defer log.CapturePanic(i.logger, &retError) - - apiName := in.APIName() - if !i.handlesMethod(apiName) { - return next(ctx, in) - } - methodName := api.MethodName(apiName) - namespaceName := MustGetNamespaceName(i.namespaceRegistry, in) - - if namespaceName != namespace.EmptyName { - var waitTime *time.Duration - defer func() { - if waitTime != nil { - metrics.HandoverWaitLatency.With(i.metricsHandler).Record(*waitTime) - } - }() - waitTime, err := i.waitNamespaceHandoverUpdate(ctx, namespaceName, methodName) - if err != nil { - metricsHandler, logTags := CreateUnaryMetricsHandlerLogTags( - i.metricsHandler, - in, - apiName, - methodName, - namespaceName, - ) - // count the request as this will not be counted - metrics.ServiceRequests.With(metricsHandler).Record(1) - - i.requestErrorHandler.HandleError( - in, - apiName, - metricsHandler, - logTags, - err, - namespaceName, - ) - return nil, err - } - } +) (any, error) { return next(ctx, in) } diff --git a/common/rpc/interceptor/namespace_validator.go b/common/rpc/interceptor/namespace_validator.go index cfd1517141d..4a4c1e02595 100644 --- a/common/rpc/interceptor/namespace_validator.go +++ b/common/rpc/interceptor/namespace_validator.go @@ -31,11 +31,10 @@ type ( additionalAllowedMethodsDuringHandover map[string]struct{} } - // NamespaceStateValidatorInterceptor validates/sets the namespace on a request and enforces - // the namespace name length limit. It is separate from NamespaceValidatorInterceptor to allow - // both to expose cleaner Intercept/InterceptNexus methods that are used as gRPC and Nexus - // interceptors. - NamespaceStateValidatorInterceptor struct { + // NamespaceLengthValidatorInterceptor enforces the namespace name length limit. It is separate + // from NamespaceValidatorInterceptor to allow both to expose cleaner Intercept/InterceptNexus + // methods that are used as gRPC and Nexus interceptors. + NamespaceLengthValidatorInterceptor struct { namespaceRegistry namespace.Registry tokenSerializer *tasktoken.Serializer maxNamespaceLength dynamicconfig.IntPropertyFn @@ -98,7 +97,7 @@ var ( ) var _ grpc.UnaryServerInterceptor = (*NamespaceValidatorInterceptor)(nil).Intercept -var _ grpc.UnaryServerInterceptor = (*NamespaceStateValidatorInterceptor)(nil).Intercept +var _ grpc.UnaryServerInterceptor = (*NamespaceLengthValidatorInterceptor)(nil).Intercept func NewNamespaceValidatorInterceptor( namespaceRegistry namespace.Registry, @@ -119,18 +118,18 @@ func NewNamespaceValidatorInterceptor( } } -func NewNamespaceStateValidatorInterceptor( +func NewNamespaceLengthValidatorInterceptor( namespaceRegistry namespace.Registry, maxNamespaceLength dynamicconfig.IntPropertyFn, -) *NamespaceStateValidatorInterceptor { - return &NamespaceStateValidatorInterceptor{ +) *NamespaceLengthValidatorInterceptor { + return &NamespaceLengthValidatorInterceptor{ namespaceRegistry: namespaceRegistry, tokenSerializer: tasktoken.NewSerializer(), maxNamespaceLength: maxNamespaceLength, } } -func (nsvi *NamespaceStateValidatorInterceptor) Intercept( +func (nsvi *NamespaceLengthValidatorInterceptor) Intercept( ctx context.Context, req any, info *grpc.UnaryServerInfo, @@ -148,7 +147,7 @@ func (nsvi *NamespaceStateValidatorInterceptor) Intercept( return handler(ctx, req) } -func (nsvi *NamespaceStateValidatorInterceptor) InterceptNexus( +func (nsvi *NamespaceLengthValidatorInterceptor) InterceptNexus( ctx context.Context, in nexus.InterceptorInput, next nexus.HandlerFunc, diff --git a/common/rpc/interceptor/namespace_validator_test.go b/common/rpc/interceptor/namespace_validator_test.go index c7d932ea884..f83d02c5715 100644 --- a/common/rpc/interceptor/namespace_validator_test.go +++ b/common/rpc/interceptor/namespace_validator_test.go @@ -901,7 +901,7 @@ func (s *namespaceValidatorSuite) Test_Intercept_SearchAttributeRequests() { } func (s *namespaceValidatorSuite) Test_NamespaceValidateIntercept() { - nnvi := NewNamespaceStateValidatorInterceptor( + nnvi := NewNamespaceLengthValidatorInterceptor( s.mockRegistry, dynamicconfig.GetIntPropertyFn(10), ) diff --git a/common/rpc/interceptor/nexus/nexus.go b/common/rpc/interceptor/nexus/nexus.go index ab229a43bdd..b669ae7dcff 100644 --- a/common/rpc/interceptor/nexus/nexus.go +++ b/common/rpc/interceptor/nexus/nexus.go @@ -25,7 +25,7 @@ type Interceptor func(ctx context.Context, in InterceptorInput, next HandlerFunc type InterceptorInput interface { ServiceName() string OperationName() string - NamespaceName() string // TODO: this should just use NamespaceEntry() instead + NamespaceName() string ForwardingInfo() ForwardingInfo APIName() string // analogous to the gRPC FullMethod NamespaceEntry() (*namespace.Namespace, error) @@ -66,7 +66,7 @@ type InterceptorError struct { } func (t *InterceptorError) Error() string { - return fmt.Sprintf("interceptor error (%s): %v", t.Outcome, t.Err.Error()) + return fmt.Sprintf("interceptor error (%s): %v", t.Outcome, t.Err) } func (t *InterceptorError) Unwrap() error { @@ -91,7 +91,7 @@ func (o *OutcomeOverride) Set(v string) { o.value = v } -func (o *OutcomeOverride) Get() string { +func (o *OutcomeOverride) Value() string { if o == nil { return "" } diff --git a/common/rpc/interceptor/nexus/nexus_test.go b/common/rpc/interceptor/nexus/nexus_test.go index 47128e887c7..e4f3b147d7e 100644 --- a/common/rpc/interceptor/nexus/nexus_test.go +++ b/common/rpc/interceptor/nexus/nexus_test.go @@ -2,6 +2,7 @@ package nexus import ( "context" + "errors" "net/http" "testing" "time" @@ -11,6 +12,74 @@ import ( "go.temporal.io/server/common/nexus/nexusrpc" ) +func TestOperationInputOutcomes(t *testing.T) { + handlerErr := nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid input") + tests := []struct { + name string + input InterceptorInput + out any + err error + outcome string + }{ + { + name: "start synchronous success", + input: StartOpInput{}, + out: &nexus.HandlerStartOperationResultSync[any]{}, + outcome: "sync_success", + }, + { + name: "start asynchronous success", + input: StartOpInput{}, + out: &nexus.HandlerStartOperationResultAsync{}, + outcome: "async_success", + }, + { + name: "start interceptor error", + input: StartOpInput{}, + err: &InterceptorError{Err: errors.New("failed"), Outcome: "custom_outcome"}, + outcome: "custom_outcome", + }, + { + name: "cancel success", + input: CancelOpInput{}, + outcome: "success", + }, + { + name: "cancel unclassified error", + input: CancelOpInput{}, + err: errors.New("failed"), + outcome: "internal_error", + }, + { + name: "completion success", + input: CompleteOpInput{}, + outcome: "success", + }, + { + name: "completion interceptor error", + input: CompleteOpInput{}, + err: &InterceptorError{Err: errors.New("failed"), Outcome: "custom_outcome"}, + outcome: "custom_outcome", + }, + { + name: "completion handler error", + input: CompleteOpInput{}, + err: handlerErr, + outcome: "error_bad_request", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.outcome, tc.input.Outcome(tc.out, tc.err)) + }) + } + + require.Equal(t, "interceptor error (): ", (&InterceptorError{}).Error()) + _, err := NewCompleteOpInput("namespace", time.Now(), nil, nil, ForwardingInfo{}, RequestMetadata{}) + require.EqualError(t, err, "nexus completion request not found") +} + func TestInterceptorInputRequest(t *testing.T) { dispatchRequest := &http.Request{Method: http.MethodPost} requestStartTime := time.Date(2026, time.May, 5, 17, 0, 0, 123456789, time.UTC) diff --git a/common/rpc/interceptor/service_error_interceptor.go b/common/rpc/interceptor/service_error_interceptor.go index 191bb73180e..b99ab3f9fcc 100644 --- a/common/rpc/interceptor/service_error_interceptor.go +++ b/common/rpc/interceptor/service_error_interceptor.go @@ -4,13 +4,14 @@ import ( "context" "errors" - nexusrpc "github.com/nexus-rpc/sdk-go/nexus" + "github.com/nexus-rpc/sdk-go/nexus" "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/persistence/serialization" - "go.temporal.io/server/common/rpc/interceptor/nexus" + interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" "go.temporal.io/server/common/util" "google.golang.org/grpc" "google.golang.org/grpc/status" @@ -51,11 +52,11 @@ func (i *ServiceErrorInterceptor) Intercept( func (i *ServiceErrorInterceptor) InterceptNexus( ctx context.Context, - in nexus.InterceptorInput, - next nexus.HandlerFunc, + in interceptornexus.InterceptorInput, + next interceptornexus.HandlerFunc, ) (any, error) { resp, err := i.capturePanicHandlerNexus(ctx, in, next) - if ie, ok := errors.AsType[*nexus.InterceptorError](err); ok { + if ie, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { ie.Err = i.transformNexusError(ie.Err) return resp, ie } @@ -95,10 +96,32 @@ func (i *ServiceErrorInterceptor) capturePanicHandler( func (i *ServiceErrorInterceptor) capturePanicHandlerNexus( ctx context.Context, - in nexus.InterceptorInput, - next nexus.HandlerFunc, + in interceptornexus.InterceptorInput, + next interceptornexus.HandlerFunc, ) (_ any, retError error) { - defer metrics.CapturePanic(i.logger, i.metricsHandler, &retError) + logTags := []tag.Tag{ + tag.Operation(in.MethodName()), + tag.WorkflowNamespace(in.NamespaceName()), + } + if endpointName := in.EndpointName(); endpointName != "" { + logTags = append(logTags, tag.Endpoint(endpointName)) + } + if operationName := in.OperationName(); operationName != "" { + logTags = append(logTags, tag.NexusOperation(operationName)) + } + switch input := in.(type) { + case interceptornexus.StartOpInput: + logTags = append(logTags, tag.NexusStageHandlerInbound, tag.RequestID(input.StartOperationOptions.RequestID)) + case interceptornexus.CancelOpInput: + logTags = append(logTags, tag.NexusStageHandlerInbound) + case interceptornexus.CompleteOpInput: + logTags = append(logTags, tag.NexusStageCallerInbound) + if input.Completion != nil && input.Completion.GetRequestId() != "" { + logTags = append(logTags, tag.RequestID(input.Completion.GetRequestId())) + } + default: + } + defer metrics.CapturePanic(log.With(i.logger, logTags...), i.metricsHandler, &retError) return next(ctx, in) } @@ -108,10 +131,10 @@ func (i *ServiceErrorInterceptor) transformNexusError(err error) error { if err == nil { return nil } - if _, ok := errors.AsType[*nexusrpc.HandlerError](err); ok { + if _, ok := errors.AsType[*nexus.HandlerError](err); ok { return err } - if _, ok := errors.AsType[*nexusrpc.OperationError](err); ok { + if _, ok := errors.AsType[*nexus.OperationError](err); ok { return err } return i.transformError(err) diff --git a/common/rpc/interceptor/telemetry.go b/common/rpc/interceptor/telemetry.go index 4011de82521..fcf675d4d8a 100644 --- a/common/rpc/interceptor/telemetry.go +++ b/common/rpc/interceptor/telemetry.go @@ -245,7 +245,7 @@ func (ti *TelemetryInterceptor) InterceptNexusOutermost( // override outcome if its set - for request forwarding cases. // error cases are captured by the wrapped InterceptorError if err == nil { - if override := outcomeOverride.Get(); override != "" { + if override := outcomeOverride.Value(); override != "" { outcome = override } } diff --git a/service/frontend/frontend_interceptors.go b/service/frontend/frontend_interceptors.go index b2342fd7682..86d21b10867 100644 --- a/service/frontend/frontend_interceptors.go +++ b/service/frontend/frontend_interceptors.go @@ -29,12 +29,12 @@ type Interceptor interface { ) (any, error) } -type InterceptorsProvider struct { +type interceptorsProvider struct { interceptors []Interceptor nexusTelemetry nexus.Interceptor // required to be first in the Nexus chain } -func NewInterceptorsProvider( +func newInterceptorsProvider( maskInternalErrorDetailsInterceptor *interceptor.MaskInternalErrorDetailsInterceptor, serviceErrorInterceptor *interceptor.ServiceErrorInterceptor, frontendServiceErrorInterceptor *interceptor.FrontendServiceErrorInterceptor, @@ -47,7 +47,7 @@ func NewInterceptorsProvider( nexusForwarder *nexusForwardingInterceptor, telemetryInterceptor *interceptor.TelemetryInterceptor, healthInterceptor *interceptor.HealthInterceptor, - namespaceStateValidatorInterceptor *interceptor.NamespaceStateValidatorInterceptor, + namespaceLengthValidatorInterceptor *interceptor.NamespaceLengthValidatorInterceptor, namespaceCountLimiterInterceptor *interceptor.ConcurrentRequestLimitInterceptor, namespaceRateLimiterInterceptorWrapper *interceptor.NamespaceRateLimitInterceptorWrapper, rateLimitInterceptor *interceptor.RateLimitInterceptor, @@ -60,7 +60,7 @@ func NewInterceptorsProvider( customInterceptors []Interceptor, retryableInterceptor *interceptor.RetryableInterceptor, faultsInterceptor *grpcfaults.FaultsInterceptor, -) *InterceptorsProvider { +) *interceptorsProvider { metricsCtxInjectorInterceptor := &interceptorWrapper{ grpcInterceptor: metrics.NewServerMetricsContextInjectorInterceptor(), @@ -83,7 +83,7 @@ func NewInterceptorsProvider( serviceErrorInterceptor, frontendServiceErrorInterceptor, businessIDInterceptor, - namespaceStateValidatorInterceptor, + namespaceLengthValidatorInterceptor, namespaceLogInterceptor, metricsCtxInjectorInterceptor, authInterceptor, @@ -112,13 +112,13 @@ func NewInterceptorsProvider( interceptors = append(interceptors, faultsInterceptor) interceptors = append(interceptors, retryableInterceptor) - return &InterceptorsProvider{ + return &interceptorsProvider{ interceptors: interceptors, nexusTelemetry: telemetryInterceptor.InterceptNexusOutermost, } } -func (n *InterceptorsProvider) GrpcInterceptors() []grpc.UnaryServerInterceptor { +func (n *interceptorsProvider) grpcInterceptors() []grpc.UnaryServerInterceptor { grpcInterceptors := make([]grpc.UnaryServerInterceptor, 0, len(n.interceptors)) for _, i := range n.interceptors { grpcInterceptors = append(grpcInterceptors, i.Intercept) @@ -126,7 +126,7 @@ func (n *InterceptorsProvider) GrpcInterceptors() []grpc.UnaryServerInterceptor return grpcInterceptors } -func (n *InterceptorsProvider) NexusInterceptors() []nexus.Interceptor { +func (n *interceptorsProvider) nexusInterceptors() []nexus.Interceptor { nexusInterceptors := make([]nexus.Interceptor, 0, len(n.interceptors)+1) // telemetry is the outermost in chain for Nexus requests to allow recording // all metrics and retain behavior. In the future, gRPC will also move telemetry diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 62a3dd6327f..10b3a7afa5d 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -105,7 +105,7 @@ var Module = fx.Options( fx.Provide(interceptor.NewHealthInterceptor), fx.Provide(NamespaceCountLimitInterceptorProvider), fx.Provide(NamespaceValidatorInterceptorProvider), - fx.Provide(NamespaceStateValidatorInterceptorProvider), + fx.Provide(NamespaceLengthValidatorInterceptorProvider), fx.Provide(NamespaceRateLimitersProvider), fx.Provide(NamespaceRateLimitInterceptorProvider), fx.Provide(SDKVersionInterceptorProvider), @@ -135,7 +135,7 @@ var Module = fx.Options( fx.Provide(newNexusForwardingInterceptor), fx.Provide(interceptor.NewNamespaceRateLimitInterceptorWrapper), fx.Provide(NewFaultsInterceptorProvider), - fx.Provide(NewInterceptorsProvider), + fx.Provide(newInterceptorsProvider), fx.Provide(newNexusCompletionHandler), fx.Provide(NewNexusOperationHTTPHandler), fx.Provide(newNexusCompletionHTTPHandler), @@ -242,7 +242,7 @@ func GrpcServerOptionsProvider( serviceConfig *Config, serviceName primitives.ServiceName, rpcFactory common.RPCFactory, - interceptorsProvider *InterceptorsProvider, + interceptorsProvider *interceptorsProvider, telemetryInterceptor *interceptor.TelemetryInterceptor, traceStatsHandler telemetry.ServerStatsHandler, metricsStatsHandler metrics.ServerStatsHandler, @@ -274,7 +274,7 @@ func GrpcServerOptionsProvider( logger.Fatal("creating gRPC server options failed", tag.Error(err)) } - unaryInterceptors := interceptorsProvider.GrpcInterceptors() + unaryInterceptors := interceptorsProvider.grpcInterceptors() streamInterceptor := []grpc.StreamServerInterceptor{ authInterceptor.InterceptStream, @@ -640,10 +640,10 @@ func NamespaceValidatorInterceptorProvider( ) } -func NamespaceStateValidatorInterceptorProvider( +func NamespaceLengthValidatorInterceptorProvider( params NamespaceValidatorInterceptorParams, -) *interceptor.NamespaceStateValidatorInterceptor { - return interceptor.NewNamespaceStateValidatorInterceptor( +) *interceptor.NamespaceLengthValidatorInterceptor { + return interceptor.NewNamespaceLengthValidatorInterceptor( params.NamespaceRegistry, params.ServiceConfig.MaxIDLengthLimit, ) diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index 003075c5be7..27811f9e2e3 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -17,7 +17,6 @@ import ( "go.temporal.io/server/api/historyservice/v1" tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/common/authorization" - "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" @@ -48,6 +47,7 @@ type nexusCompletionHandler struct { CallbackTokenGenerator *commonnexus.CallbackTokenGenerator HistoryClient resource.HistoryClient RequestErrorHandler *interceptor.RequestErrorHandler + telemetryInterceptor *interceptor.TelemetryInterceptor AuthInterceptor *authorization.Interceptor // required for parsing auth info, not used as an interceptor preProcessErrorsCounter metrics.CounterIface chainedHandler interceptornexus.HandlerFunc @@ -58,7 +58,6 @@ type nexusCompletionHTTPHandler struct { } func newNexusCompletionHandler( - clusterMetadata cluster.Metadata, namespaceRegistry namespace.Registry, logger log.Logger, metricsHandler metrics.Handler, @@ -67,8 +66,8 @@ func newNexusCompletionHandler( historyClient resource.HistoryClient, requestErrorHandler *interceptor.RequestErrorHandler, authInterceptor *authorization.Interceptor, - httpTraceProvider commonnexus.HTTPClientTraceProvider, - interceptorsProvider *InterceptorsProvider, + telemetryInterceptor *interceptor.TelemetryInterceptor, + interceptorsProvider *interceptorsProvider, ) *nexusCompletionHandler { h := &nexusCompletionHandler{ @@ -80,9 +79,10 @@ func newNexusCompletionHandler( HistoryClient: historyClient, RequestErrorHandler: requestErrorHandler, AuthInterceptor: authInterceptor, + telemetryInterceptor: telemetryInterceptor, preProcessErrorsCounter: metricsHandler.Counter(metrics.NexusCompletionRequestPreProcessErrors.Name()), } - h.chainedHandler = interceptornexus.ChainInterceptors(h.finalCompleteHandler, interceptorsProvider.NexusInterceptors()) + h.chainedHandler = interceptornexus.ChainInterceptors(h.finalCompleteHandler, interceptorsProvider.nexusInterceptors()) return h } @@ -154,12 +154,23 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus ctx = rCtx.augmentContext(ctx, r.HTTPRequest.Header) defer finalizeCompletionRequest(rCtx, &retErr) - // recordBadRequest is for pre-interceptor chain error recording - recordBadRequest := func() { - metrics.NexusCompletionRequests.With(h.MetricsHandler).Record( - 1, + const outcomeBadRequest = "error_bad_request" + + // recordPreInterceptorFailure is for pre-interceptor chain error recording. + recordPreInterceptorFailure := func(outcome string) { + completionMetrics := h.MetricsHandler.WithTags( metrics.NamespaceTag(ns.Name().String()), - metrics.OutcomeTag("error_bad_request"), + metrics.OutcomeTag(outcome), + ) + completionMetrics.Counter(metrics.NexusCompletionRequests.Name()).Record(1) + completionMetrics.Histogram( + metrics.NexusCompletionLatencyHistogram.Name(), + metrics.Milliseconds, + ).Record(time.Since(requestStartTime).Milliseconds()) + + metrics.ServiceRequests.With(rCtx.metricsHandlerForInterceptors).Record(1) + h.telemetryInterceptor.RecordLatencyMetrics( + ctx, requestStartTime, rCtx.metricsHandlerForInterceptors, ) } @@ -169,7 +180,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus if err != nil { logger.Error("failed to extract namespace from request", tag.Error(err)) h.preProcessErrorsCounter.Record(1) - recordBadRequest() + recordPreInterceptorFailure(outcomeBadRequest) return &interceptornexus.InterceptorError{ Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid URL"), SkipServiceErrorReporting: true, @@ -180,7 +191,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus "namespace in callback URL doesn't match the completion token", tag.String("url-namespace", nsName), ) - recordBadRequest() + recordPreInterceptorFailure(outcomeBadRequest) return &interceptornexus.InterceptorError{ Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid callback token"), SkipServiceErrorReporting: true, @@ -189,7 +200,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus } ctx, err = rCtx.parseTLSAndAuthInfo(ctx, r) if err != nil { - recordBadRequest() + recordPreInterceptorFailure("error_internal") return &interceptornexus.InterceptorError{ Err: err, SkipServiceErrorReporting: true, @@ -212,7 +223,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus ) if err != nil { logger.Error("invalid nexus completion request", tag.Error(err)) - recordBadRequest() + recordPreInterceptorFailure(outcomeBadRequest) return &interceptornexus.InterceptorError{ Err: nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid request"), SkipServiceErrorReporting: true, @@ -278,7 +289,13 @@ func (h *nexusCompletionHandler) finalCompleteHandler( if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "error_not_found", ExposeDetails: true} } - return nil, &interceptornexus.InterceptorError{Err: err, Outcome: "error_internal"} + // Preserve specific outcome tags on handler errors. + converted := commonnexus.ConvertGRPCError(err, false) + outcome := "error_internal" + if handlerErr, ok := errors.AsType[*nexus.HandlerError](converted); ok { + outcome = "error_" + strings.ToLower(string(handlerErr.Type)) + } + return nil, &interceptornexus.InterceptorError{Err: err, Outcome: outcome} } // completeOperation dispatches the completion to the framework named by its diff --git a/service/frontend/nexus_dispatch_result.go b/service/frontend/nexus_dispatch_result.go index 3bb104cfb4a..fee7720fbef 100644 --- a/service/frontend/nexus_dispatch_result.go +++ b/service/frontend/nexus_dispatch_result.go @@ -21,7 +21,7 @@ func (c *operationContext) handleStartOperationResponse( operation string, ) (nexus.HandlerStartOperationResult[any], []nexus.Link, error) { result := commonnexus.ClassifyStartOperationDispatch(resp) - c.recordDispatchOutcome(result) + c.attributeFailureToWorker(result) switch result.Outcome { case commonnexus.DispatchOutcomeSyncSuccess: @@ -60,7 +60,7 @@ func (c *operationContext) handleCancelOperationResponse( operation string, ) error { result := commonnexus.ClassifyCancelOperationDispatch(resp) - c.recordDispatchOutcome(result) + c.attributeFailureToWorker(result) if result.Outcome == commonnexus.DispatchOutcomeCancelAccepted { return nil @@ -153,9 +153,7 @@ func (c *operationContext) operationError( return opErr } -// recordDispatchOutcome attributes a failed dispatch to the worker in the response header. The -// outcome is carried by the returned InterceptorError and recorded by the Nexus telemetry interceptor. -func (c *operationContext) recordDispatchOutcome(result commonnexus.DispatchResult) { +func (c *operationContext) attributeFailureToWorker(result commonnexus.DispatchResult) { if !result.Outcome.Succeeded() { c.setFailureSource(commonnexus.FailureSourceWorker) } diff --git a/service/frontend/nexus_dispatch_result_test.go b/service/frontend/nexus_dispatch_result_test.go index ecea95228f5..3cbba35f029 100644 --- a/service/frontend/nexus_dispatch_result_test.go +++ b/service/frontend/nexus_dispatch_result_test.go @@ -24,7 +24,7 @@ import ( // These tests pin down how the frontend turns matching's DispatchNexusTaskResponse into the result the // Nexus SDK serializes back to the caller. Every arm of the response oneof is wire-visible: the error // type decides the HTTP status, and the outcome tag and failure-source header are consumed by -// dashboards and by interceptRequest's error-reporting cleanup. They are asserted here so the shared +// dashboards and by request error-reporting cleanup. They are asserted here so the shared // classifier introduced alongside them cannot silently change any of it. func failureSourceOf(oc *operationContext) string { @@ -65,10 +65,6 @@ func requireRecordedDispatchOutcome( require.Equal(t, expectedOutcome, snapshot[metrics.NexusRequests.Name()][0].Tags[outcomeTag.Key]) } -func testOperationContext() *operationContext { - return newOperationContext() -} - // startOperationResponse wraps a StartOperationResponse in the matching response envelope. The oneof // variant interface is unexported, so callers pass a fully built StartOperationResponse. func startOperationResponse(sor *nexuspb.StartOperationResponse) *matchingservice.DispatchNexusTaskResponse { diff --git a/service/frontend/nexus_forward_interceptor.go b/service/frontend/nexus_forward_interceptor.go index f6a768b594a..c7c23413b06 100644 --- a/service/frontend/nexus_forward_interceptor.go +++ b/service/frontend/nexus_forward_interceptor.go @@ -11,6 +11,7 @@ import ( "github.com/nexus-rpc/sdk-go/nexus" "go.temporal.io/server/common" + "go.temporal.io/server/common/api" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" @@ -83,10 +84,8 @@ func (i *nexusForwardingInterceptor) InterceptNexus( interceptornexus.SetOutcomeOverride(ctx, "request_forwarded") - // this is the user-facing operation identity, and the DCRedirection prefix - // matches the convention the gRPC redirection path uses for the same metrics. metricsHandler, forwardStartTime := i.redirectionInterceptor.BeforeCall( - interceptor.DCRedirectionMetricsPrefix + in.MethodName(), + interceptor.DCRedirectionMetricsPrefix + api.MethodName(in.APIName()), ) defer func() { redirectionErr := retErr @@ -110,7 +109,14 @@ func (i *nexusForwardingInterceptor) InterceptNexus( // empty for completion requests logTags = append(logTags, tag.NexusOperation(operationName)) } - logger := log.With(i.logger, logTags...) + // Retrieve loggers for operation type-specific tags. + baseLogger := i.logger + if rCtx, ok := requestContextFromContext(ctx); ok { + baseLogger = rCtx.logger + } else if oc, ok := operationContextFromContext(ctx); ok { + baseLogger = oc.logger + } + logger := log.With(baseLogger, logTags...) switch request := in.(type) { case interceptornexus.StartOpInput: diff --git a/service/frontend/nexus_forward_interceptor_test.go b/service/frontend/nexus_forward_interceptor_test.go index e3caef35b03..405aa48d361 100644 --- a/service/frontend/nexus_forward_interceptor_test.go +++ b/service/frontend/nexus_forward_interceptor_test.go @@ -192,6 +192,7 @@ func TestNexusForwardingInterceptorInterceptNexus(t *testing.T) { require.IsType(t, &nexus.HandlerStartOperationResultAsync{}, result) require.Equal(t, "true", receivedHeaders.Get(interceptor.DCRedirectionAPIHeaderName)) require.Equal(t, currentCluster, receivedHeaders.Get(interceptor.DCRedirectionSourceCellHeaderName)) + require.Equal(t, "original", receivedHeaders.Get("X-Original")) case requestFailed: require.Nil(t, result) default: diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index bc9a04c5a16..14d6c52d483 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -20,7 +20,6 @@ import ( taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/server/api/matchingservice/v1" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" - "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" @@ -114,6 +113,9 @@ func (c *operationContext) handleRequestError(err error) { return } if taggedErr, ok := errors.AsType[*interceptornexus.InterceptorError](err); ok { + if taggedErr.SkipServiceErrorReporting { + return + } err = taggedErr.Err } source, ok := c.responseHeaders[commonnexus.FailureSourceHeaderName] @@ -213,7 +215,6 @@ type nexusHandler struct { nexus.UnimplementedHandler logger log.Logger metricsHandler metrics.Handler - clusterMetadata cluster.Metadata namespaceRegistry namespace.Registry matchingClient matchingservice.MatchingServiceClient requestErrorHandler *interceptor.RequestErrorHandler @@ -226,7 +227,6 @@ type nexusHandler struct { func newNexusHandler( logger log.Logger, metricsHandler metrics.Handler, - clusterMetadata cluster.Metadata, namespaceRegistry namespace.Registry, matchingClient matchingservice.MatchingServiceClient, requestErrorHandler *interceptor.RequestErrorHandler, @@ -238,7 +238,6 @@ func newNexusHandler( h := &nexusHandler{ logger: logger, metricsHandler: metricsHandler, - clusterMetadata: clusterMetadata, namespaceRegistry: namespaceRegistry, matchingClient: matchingClient, requestErrorHandler: requestErrorHandler, diff --git a/service/frontend/nexus_handler_test.go b/service/frontend/nexus_handler_test.go index 6d28ca3c533..15ce9338f45 100644 --- a/service/frontend/nexus_handler_test.go +++ b/service/frontend/nexus_handler_test.go @@ -11,7 +11,7 @@ import ( "go.temporal.io/server/common/primitives/timestamp" ) -func newOperationContext() *operationContext { +func testOperationContext() *operationContext { oc := &operationContext{ nexusContext: &nexusContext{}, } diff --git a/service/frontend/nexus_interceptor_chain_test.go b/service/frontend/nexus_interceptor_chain_test.go index 4050757bbce..3df3f5ce2d3 100644 --- a/service/frontend/nexus_interceptor_chain_test.go +++ b/service/frontend/nexus_interceptor_chain_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "testing" "time" @@ -15,17 +16,89 @@ import ( "go.temporal.io/server/common/metrics/metricstest" rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" interceptornexus "go.temporal.io/server/common/rpc/interceptor/nexus" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) +type testGRPCError struct { + status *status.Status +} + +func (e testGRPCError) Error() string { + return e.status.Message() +} + +func (e testGRPCError) GRPCStatus() *status.Status { + return e.status +} + +func TestInterceptorsProviderOrder(t *testing.T) { + customGRPCInterceptor := func(context.Context, any, *grpc.UnaryServerInfo, grpc.UnaryHandler) (any, error) { + return nil, nil + } + customInterceptor := &interceptorWrapper{ + grpcInterceptor: customGRPCInterceptor, + nexusInterceptor: nexusNoOpInterceptor, + } + provider := newInterceptorsProvider( + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + []grpc.UnaryServerInterceptor{customGRPCInterceptor}, []Interceptor{customInterceptor}, nil, nil, + ) + + expectedTypes := []string{ + "*interceptor.MaskInternalErrorDetailsInterceptor", + "*interceptor.ServiceErrorInterceptor", + "*interceptor.FrontendServiceErrorInterceptor", + "*interceptor.RoutingKeyInterceptor", + "*interceptor.NamespaceLengthValidatorInterceptor", + "*interceptor.NamespaceLogInterceptor", + "*frontend.interceptorWrapper", + "*authorization.Interceptor", + "*interceptor.NamespaceHandoverInterceptor", + "*frontend.interceptorWrapper", + "*interceptor.TelemetryInterceptor", + "*interceptor.HealthInterceptor", + "*interceptor.NamespaceValidatorInterceptor", + "*interceptor.ConcurrentRequestLimitInterceptor", + "*interceptor.NamespaceRateLimitInterceptorWrapper", + "*interceptor.RateLimitInterceptor", + "*interceptor.SDKVersionInterceptor", + "*interceptor.CallerInfoInterceptor", + "*interceptor.SlowRequestLoggerInterceptor", + "*chasm.ChasmVisibilityInterceptor", + "*interceptor.ContextMetadataInterceptor", + "*frontend.interceptorWrapper", + "*frontend.interceptorWrapper", + "*grpcfaults.FaultsInterceptor", + "*interceptor.RetryableInterceptor", + } + actualTypes := make([]string, 0, len(provider.interceptors)) + for _, current := range provider.interceptors { + actualTypes = append(actualTypes, reflect.TypeOf(current).String()) + } + require.Equal(t, expectedTypes, actualTypes) + + grpcInterceptors := provider.grpcInterceptors() + nexusInterceptors := provider.nexusInterceptors() + require.Len(t, grpcInterceptors, len(expectedTypes)) + require.Len(t, nexusInterceptors, len(grpcInterceptors)+1) + require.Equal( + t, + reflect.ValueOf(provider.nexusTelemetry).Pointer(), + reflect.ValueOf(nexusInterceptors[0]).Pointer(), + "Outermost interceptor for Nexus must be telemetry", + ) +} + func TestNexusChainPreservesNativeErrors(t *testing.T) { tests := []struct { - name string - err error - outcome string - wrapError bool - assertErrors func(*testing.T, error, bool) + name string + err error + outcome string + wrapError bool + exposeDetails bool + assertErrors func(*testing.T, error, bool) }{ { name: "operation error", @@ -93,6 +166,31 @@ func TestNexusChainPreservesNativeErrors(t *testing.T) { require.Equal(t, "internal error", handlerErr.Message) }, }, + { + name: "resource exhausted error details", + err: testGRPCError{status: status.New(codes.ResourceExhausted, "namespace rate limit exceeded")}, + outcome: "namespace_rate_limited", + wrapError: true, + exposeDetails: true, + assertErrors: func(t *testing.T, err error, _ bool) { + var handlerErr *nexus.HandlerError + require.ErrorAs(t, convertInterceptorError(err), &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeResourceExhausted, handlerErr.Type) + require.Contains(t, handlerErr.Message, "namespace rate limit exceeded") + }, + }, + { + name: "resource exhausted error details masked", + err: testGRPCError{status: status.New(codes.ResourceExhausted, "namespace rate limit exceeded")}, + outcome: "namespace_rate_limited", + wrapError: true, + assertErrors: func(t *testing.T, err error, _ bool) { + var handlerErr *nexus.HandlerError + require.ErrorAs(t, convertInterceptorError(err), &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeResourceExhausted, handlerErr.Type) + require.Equal(t, "resource exhausted", handlerErr.Message) + }, + }, } for _, tc := range tests { @@ -104,17 +202,15 @@ func TestNexusChainPreservesNativeErrors(t *testing.T) { capture := metricsHandler.StartCapture() defer metricsHandler.StopCapture(capture) - chainedHandler := newTestNexusInterceptorChain(metricsHandler, maskErrors, tc.err, tc.outcome, tc.wrapError) + chainedHandler := newTestNexusInterceptorChain(metricsHandler, maskErrors, tc.err, tc.outcome, tc.wrapError, tc.exposeDetails) _, err := chainedHandler(context.Background(), newTestNexusStartInput()) - rawErr := err if tc.wrapError { var interceptorErr *interceptornexus.InterceptorError require.ErrorAs(t, err, &interceptorErr) require.Equal(t, tc.outcome, interceptorErr.Outcome) - rawErr = interceptorErr.Err } - tc.assertErrors(t, rawErr, maskErrors) + tc.assertErrors(t, err, maskErrors) snapshot := capture.Snapshot() require.Len(t, snapshot[metrics.NexusRequests.Name()], 1) @@ -131,6 +227,7 @@ func newTestNexusInterceptorChain( terminalErr error, outcome string, wrapError bool, + exposeDetails bool, ) interceptornexus.HandlerFunc { telemetry := rpcinterceptor.NewTelemetryInterceptor(nil, metricsHandler, log.NewNoopLogger(), nil, nil) mask := rpcinterceptor.NewMaskInternalErrorDetailsInterceptor( @@ -150,7 +247,7 @@ func newTestNexusInterceptorChain( if !wrapError { return nil, terminalErr } - return nil, &interceptornexus.InterceptorError{Err: terminalErr, Outcome: outcome} + return nil, &interceptornexus.InterceptorError{Err: terminalErr, Outcome: outcome, ExposeDetails: exposeDetails} }, []interceptornexus.Interceptor{ telemetry.InterceptNexusOutermost, diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index fb3fa3e5302..62e43430c94 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -16,7 +16,6 @@ import ( "go.temporal.io/server/api/matchingservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/authorization" - "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" @@ -51,13 +50,12 @@ func NewNexusOperationHTTPHandler( serviceConfig *Config, matchingClient resource.MatchingClient, metricsHandler metrics.Handler, - clusterMetadata cluster.Metadata, namespaceRegistry namespace.Registry, endpointRegistry commonnexus.EndpointRegistry, authInterceptor *authorization.Interceptor, namespaceValidationInterceptor *interceptor.NamespaceValidatorInterceptor, requestErrorHandler *interceptor.RequestErrorHandler, - interceptorsProvider *InterceptorsProvider, + interceptorsProvider *interceptorsProvider, logger log.Logger, httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter, ) *NexusOperationHTTPHandler { @@ -79,14 +77,13 @@ func NewNexusOperationHTTPHandler( Handler: newNexusHandler( logger, metricsHandler, - clusterMetadata, namespaceRegistry, matchingservice.MatchingServiceClient(matchingClient), requestErrorHandler, serviceConfig.BlobSizeLimitError, serviceConfig.NexusRequestHeadersBlacklist, serviceConfig.NexusOperationsMetricTagConfig, - interceptorsProvider.NexusInterceptors(), + interceptorsProvider.nexusInterceptors(), ), GetResultTimeout: serviceConfig.KeepAliveMaxConnectionIdle(), Logger: log.NewSlogLogger(logger), diff --git a/temporal/server_option.go b/temporal/server_option.go index 390e56fc7f3..1edd06cda6b 100644 --- a/temporal/server_option.go +++ b/temporal/server_option.go @@ -203,7 +203,7 @@ func WithSearchAttributesMapper(m searchattribute.Mapper) ServerOption { // ServerInterceptors. The custom interceptors will be invoked in the order as they appear in the supplied list, after // the internal ServerInterceptors. // -// Deprecated: Use WithChainedFrontendInterceptors instead. +// Deprecated: Use [WithChainedFrontendInterceptors] instead. These options are mutually exclusive. func WithChainedFrontendGrpcInterceptors( interceptors ...grpc.UnaryServerInterceptor, ) ServerOption { @@ -215,6 +215,7 @@ func WithChainedFrontendGrpcInterceptors( // WithChainedFrontendInterceptors sets an ordered chain of custom gRPC+Nexus interceptors that will be invoked for all // Frontend gRPC and Nexus API calls respectively. Custom interceptors run after the internal // interceptors and before the fault-injection and retryable interceptors, in the order supplied. +// Cannot be used with [WithChainedFrontendGrpcInterceptors]- they are mutually exclusive. func WithChainedFrontendInterceptors( interceptors ...frontend.Interceptor, ) ServerOption { diff --git a/temporal/server_options.go b/temporal/server_options.go index bb192d79bb3..8ad988b7503 100644 --- a/temporal/server_options.go +++ b/temporal/server_options.go @@ -136,7 +136,7 @@ func (so *serverOptions) validateConfig() error { len(so.customFrontendUnifiedInterceptors) > 0 { // Both could be supported as a migration path but intentionally avoided as // migration itself is as simple as wrapping with no-op Nexus Interceptors. - return errors.New("configure either custom gRPC or unified interceptors, not both") + return errors.New("WithChainedFrontendGrpcInterceptors is deprecated in favor of WithChainedFrontendInterceptors- they cannot both be set") } if err := so.config.Validate(); err != nil { return err diff --git a/temporal/server_options_test.go b/temporal/server_options_test.go new file mode 100644 index 00000000000..ea279076ed9 --- /dev/null +++ b/temporal/server_options_test.go @@ -0,0 +1,46 @@ +package temporal + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/common/rpc/interceptor/nexus" + "go.temporal.io/server/service/frontend" + "google.golang.org/grpc" +) + +type testFrontendInterceptor struct{} + +func (testFrontendInterceptor) Intercept( + context.Context, + any, + *grpc.UnaryServerInfo, + grpc.UnaryHandler, +) (any, error) { + return nil, nil +} + +func (testFrontendInterceptor) InterceptNexus( + context.Context, + nexus.InterceptorInput, + nexus.HandlerFunc, +) (any, error) { + return nil, nil +} + +var _ frontend.Interceptor = testFrontendInterceptor{} + +func TestServerOptionsRejectsBothFrontendInterceptorOptions(t *testing.T) { + options := serverOptions{ + customFrontendInterceptors: []grpc.UnaryServerInterceptor{ + func(context.Context, any, *grpc.UnaryServerInfo, grpc.UnaryHandler) (any, error) { + return nil, nil + }, + }, + customFrontendUnifiedInterceptors: []frontend.Interceptor{testFrontendInterceptor{}}, + } + + err := options.validateConfig() + require.EqualError(t, err, "WithChainedFrontendGrpcInterceptors is deprecated in favor of WithChainedFrontendInterceptors- they cannot both be set") +}