diff --git a/chasm/lib/activity/activity_test.go b/chasm/lib/activity/activity_test.go index 5d3674c9dd7..fe75eef8c6a 100644 --- a/chasm/lib/activity/activity_test.go +++ b/chasm/lib/activity/activity_test.go @@ -1149,8 +1149,8 @@ func TestEffectiveUserMetadata_FallsBackToLegacy(t *testing.T) { require.Same(t, legacyMD, got) } -// A Stream ref with an activity id routes to the Activity's component, so the stream library's -// copy of the archetype must stay this library's. +// A Stream ref with an activity id routes to stream.ActivityArchetypeID, so that id must equal +// this library's ArchetypeID. func TestStreamHostArchetypeIsTheActivity(t *testing.T) { require.Equal(t, ArchetypeID, stream.ActivityArchetypeID) } diff --git a/chasm/lib/activity/stream.go b/chasm/lib/activity/stream.go index 6634b859960..d5b5f4cc9db 100644 --- a/chasm/lib/activity/stream.go +++ b/chasm/lib/activity/stream.go @@ -8,9 +8,8 @@ import ( "go.temporal.io/server/common/tasktoken" ) -// A Standalone Activity owns its Streams. They nest in the Activity's execution, so a call routes -// by the activity id, and they close in the transition that ends the Activity, with its status. -// A retry is the same execution, so a Stream goes on across attempts. +// A Standalone Activity's Streams nest in its execution, so a call routes by the activity id. A +// retry is the same execution, so a Stream stays open across attempts. var _ stream.Host = (*Activity)(nil) @@ -22,8 +21,8 @@ func (a *Activity) NestedStreams() chasm.Map[string, *stream.Stream] { return a.Streams } -// CheckOwner refuses a create that isn't the Activity's own, by its current attempt. It -// implements [stream.Host]. +// CheckOwner refuses a create unless the owner is the Activity's own Producer with the task token +// of its current attempt. It implements [stream.Host]. func (a *Activity) CheckOwner( ctx chasm.Context, owner *streamservice.Producer, @@ -35,8 +34,8 @@ func (a *Activity) CheckOwner( return a.checkAttempt(ctx, owner, token) } -// CheckWriter refuses a write as the Activity's own Producer by any other than its current -// attempt. Other Producers write as on any Stream. It implements [stream.Host]. +// CheckWriter refuses a write as the Activity's own Producer unless the task token is the +// Activity's current attempt. It doesn't check other Producers. It implements [stream.Host]. func (a *Activity) CheckWriter( ctx chasm.Context, producer *streamservice.Producer, @@ -48,7 +47,7 @@ func (a *Activity) CheckWriter( return a.checkAttempt(ctx, producer, token) } -// ownProducer says whether a Producer is this Activity's, by the id that the crate makes. +// ownProducer says whether a Producer has the kind and the id that the crate gives this Activity. func (a *Activity) ownProducer(ctx chasm.Context, producer *streamservice.Producer) bool { key := ctx.ExecutionKey() id := "sa:" + stream.EncodeIDPart(key.BusinessID) + ":" + stream.EncodeIDPart(key.RunID) @@ -57,7 +56,7 @@ func (a *Activity) ownProducer(ctx chasm.Context, producer *streamservice.Produc } // checkAttempt checks that a task token is this Activity's current attempt, and that the -// Producer's epoch is that attempt. The epoch stays (attempt scheduled time, attempt). +// Producer's epoch has the same attempt. func (a *Activity) checkAttempt( ctx chasm.Context, producer *streamservice.Producer, @@ -81,7 +80,13 @@ func (a *Activity) checkAttempt( return nil } -// endStreams closes the Activity's Streams with its end. Each terminal transition passes its +// HoldsStreams says whether a Stream of the closed Activity is still readable, so the Activity +// must outlive the namespace retention. A closed Stream stays readable for its Retention. +func (a *Activity) HoldsStreams(ctx chasm.Context) bool { + return stream.HoldsStreams(ctx, a.Streams) +} + +// endStreams closes the Activity's Streams when it ends. Each terminal transition passes its // status, since the transition sets the Activity's status after its apply function. func (a *Activity) endStreams(ctx chasm.MutableContext, status activitypb.ActivityExecutionStatus) { if len(a.Streams) == 0 { diff --git a/chasm/lib/stream/config.go b/chasm/lib/stream/config.go index af1f9908d14..8e6e6696953 100644 --- a/chasm/lib/stream/config.go +++ b/chasm/lib/stream/config.go @@ -31,7 +31,8 @@ Stream that a client or an Activity created lives one Retention after its last w "stream.maxStoredBytes", 4*1024*1024, `The most Record bytes a Stream keeps. The Records are in the Stream's mutable state, so an -append past the cap drops the oldest batches, as the Retention does.`, +append past the cap drops the oldest batches, as the Retention does. The append fails when an open +Workflow reader still needs them.`, ) MaxWorkflowStateBytes = dynamicconfig.NewNamespaceIntSetting( @@ -57,15 +58,15 @@ retryable error, so the caller backs off.`, MaxWorkflowReaders = dynamicconfig.NewNamespaceIntSetting( "stream.maxWorkflowReaders", 100, - `The most open Workflow runs whose read floor one Workflow Stream keeps. A page of a new -reader past the cap fails with a retryable error, so the caller backs off.`, + `The most open Workflow runs whose read floor one Stream keeps. A page of a new reader past +the cap fails with a retryable error, so the caller backs off.`, ) ReaderCheckInterval = dynamicconfig.NewNamespaceDurationSetting( "stream.readerCheckInterval", 5*time.Minute, - `How often a Workflow Stream with read floors checks whether their runs are still open. The -floor of a closed run goes at the next check.`, + `How often a Stream with read floors checks whether their runs are still open. The floor of +a closed run goes at the next check.`, ) MaxWakeWait = dynamicconfig.NewNamespaceDurationSetting( diff --git a/chasm/lib/stream/errors.go b/chasm/lib/stream/errors.go index 9f7b98aef49..b1ab06c883a 100644 --- a/chasm/lib/stream/errors.go +++ b/chasm/lib/stream/errors.go @@ -48,7 +48,8 @@ var reasons = []string{ // StreamErrorDetails in the status details. func NewError(reason string, format string, args ...any) error { st := status.New(reasonCode(reason), reason+": "+fmt.Sprintf(format, args...)) - if detailed, err := st.WithDetails(&streamservice.StreamErrorDetails{Reason: reason}); err == nil { + detailed, err := st.WithDetails(&streamservice.StreamErrorDetails{Reason: reason}) + if err == nil { st = detailed } return serviceerror.FromStatus(st) diff --git a/chasm/lib/stream/frontend.go b/chasm/lib/stream/frontend.go index 9dae42dda4e..95978ea2c12 100644 --- a/chasm/lib/stream/frontend.go +++ b/chasm/lib/stream/frontend.go @@ -28,16 +28,26 @@ func NewFrontendHandler( return &FrontendHandler{client: client, config: config, namespaceRegistry: namespaceRegistry} } -// RedirectableMethods lists the StreamService methods that a cell forwards to the namespace's -// active cell, with the response each one answers. The Stream is in the active cell's -// mutable state, so a call served elsewhere would read a replica or write a copy. +// RedirectableMethods maps each StreamService method that a cell forwards to the namespace's +// active cell to a func that makes its empty response. The Stream is in the active cell's mutable +// state, so a call served elsewhere would read a replica or write a copy. func RedirectableMethods() map[string]func() any { return map[string]func() any{ - streamservice.StreamService_CreateStream_FullMethodName: func() any { return &streamservice.CreateStreamResponse{} }, - streamservice.StreamService_AppendRecords_FullMethodName: func() any { return &streamservice.AppendRecordsResponse{} }, - streamservice.StreamService_CloseStream_FullMethodName: func() any { return &streamservice.CloseStreamResponse{} }, - streamservice.StreamService_ReadRecords_FullMethodName: func() any { return &streamservice.ReadRecordsResponse{} }, - streamservice.StreamService_DescribeStream_FullMethodName: func() any { return &streamservice.DescribeStreamResponse{} }, + streamservice.StreamService_CreateStream_FullMethodName: func() any { + return &streamservice.CreateStreamResponse{} + }, + streamservice.StreamService_AppendRecords_FullMethodName: func() any { + return &streamservice.AppendRecordsResponse{} + }, + streamservice.StreamService_CloseStream_FullMethodName: func() any { + return &streamservice.CloseStreamResponse{} + }, + streamservice.StreamService_ReadRecords_FullMethodName: func() any { + return &streamservice.ReadRecordsResponse{} + }, + streamservice.StreamService_DescribeStream_FullMethodName: func() any { + return &streamservice.DescribeStreamResponse{} + }, } } @@ -70,7 +80,10 @@ func (h *FrontendHandler) CreateStream( if err != nil { return nil, err } - resp, err := h.client.CreateStream(ctx, &streampb.CreateStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.CreateStream( + ctx, + &streampb.CreateStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } @@ -82,7 +95,10 @@ func (h *FrontendHandler) AppendRecords( if err != nil { return nil, err } - resp, err := h.client.AppendRecords(ctx, &streampb.AppendRecordsRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.AppendRecords( + ctx, + &streampb.AppendRecordsRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } @@ -94,7 +110,10 @@ func (h *FrontendHandler) CloseStream( if err != nil { return nil, err } - resp, err := h.client.CloseStream(ctx, &streampb.CloseStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.CloseStream( + ctx, + &streampb.CloseStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } @@ -106,7 +125,10 @@ func (h *FrontendHandler) ReadRecords( if err != nil { return nil, err } - resp, err := h.client.ReadRecords(ctx, &streampb.ReadRecordsRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.ReadRecords( + ctx, + &streampb.ReadRecordsRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } @@ -118,6 +140,9 @@ func (h *FrontendHandler) DescribeStream( if err != nil { return nil, err } - resp, err := h.client.DescribeStream(ctx, &streampb.DescribeStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.DescribeStream( + ctx, + &streampb.DescribeStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } diff --git a/chasm/lib/stream/gen/streampb/v1/message.pb.go b/chasm/lib/stream/gen/streampb/v1/message.pb.go index 1db28d19bf0..a8930386a8b 100644 --- a/chasm/lib/stream/gen/streampb/v1/message.pb.go +++ b/chasm/lib/stream/gen/streampb/v1/message.pb.go @@ -665,7 +665,7 @@ type ParkedReadState struct { Request *v1.ReadRecordsRequest `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` // The Nexus request id, so a retry of the start finds this read. RequestId string `protobuf:"bytes,2,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` - // Where the next look starts. A look that finds only other Topics moves it on. + // Where the next look starts. A look that finds only other Topics advances it. ScanOffset int64 `protobuf:"varint,3,opt,name=scan_offset,json=scanOffset,proto3" json:"scan_offset,omitempty"` // Set while the read waits for the create. CreateDeadline *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=create_deadline,json=createDeadline,proto3" json:"create_deadline,omitempty"` diff --git a/chasm/lib/stream/grpc.go b/chasm/lib/stream/grpc.go index 230e38c69e1..b1124a4ca21 100644 --- a/chasm/lib/stream/grpc.go +++ b/chasm/lib/stream/grpc.go @@ -17,7 +17,10 @@ func newGRPCHandler(h *handler) *grpcHandler { return &grpcHandler{handler: h} } -func (g *grpcHandler) CreateStream(ctx context.Context, req *streampb.CreateStreamRequest) (*streampb.CreateStreamResponse, error) { +func (g *grpcHandler) CreateStream( + ctx context.Context, + req *streampb.CreateStreamRequest, +) (*streampb.CreateStreamResponse, error) { resp, err := g.handler.CreateStream(ctx, req.GetFrontendRequest()) if err != nil { return nil, err @@ -25,7 +28,10 @@ func (g *grpcHandler) CreateStream(ctx context.Context, req *streampb.CreateStre return &streampb.CreateStreamResponse{FrontendResponse: resp}, nil } -func (g *grpcHandler) AppendRecords(ctx context.Context, req *streampb.AppendRecordsRequest) (*streampb.AppendRecordsResponse, error) { +func (g *grpcHandler) AppendRecords( + ctx context.Context, + req *streampb.AppendRecordsRequest, +) (*streampb.AppendRecordsResponse, error) { resp, err := g.handler.AppendRecords(ctx, req.GetFrontendRequest()) if err != nil { return nil, err @@ -33,7 +39,10 @@ func (g *grpcHandler) AppendRecords(ctx context.Context, req *streampb.AppendRec return &streampb.AppendRecordsResponse{FrontendResponse: resp}, nil } -func (g *grpcHandler) CloseStream(ctx context.Context, req *streampb.CloseStreamRequest) (*streampb.CloseStreamResponse, error) { +func (g *grpcHandler) CloseStream( + ctx context.Context, + req *streampb.CloseStreamRequest, +) (*streampb.CloseStreamResponse, error) { resp, err := g.handler.CloseStream(ctx, req.GetFrontendRequest()) if err != nil { return nil, err @@ -41,7 +50,10 @@ func (g *grpcHandler) CloseStream(ctx context.Context, req *streampb.CloseStream return &streampb.CloseStreamResponse{FrontendResponse: resp}, nil } -func (g *grpcHandler) ReadRecords(ctx context.Context, req *streampb.ReadRecordsRequest) (*streampb.ReadRecordsResponse, error) { +func (g *grpcHandler) ReadRecords( + ctx context.Context, + req *streampb.ReadRecordsRequest, +) (*streampb.ReadRecordsResponse, error) { resp, err := g.handler.ReadRecords(ctx, req.GetFrontendRequest(), nil) if err != nil { return nil, err @@ -49,7 +61,10 @@ func (g *grpcHandler) ReadRecords(ctx context.Context, req *streampb.ReadRecords return &streampb.ReadRecordsResponse{FrontendResponse: resp}, nil } -func (g *grpcHandler) DescribeStream(ctx context.Context, req *streampb.DescribeStreamRequest) (*streampb.DescribeStreamResponse, error) { +func (g *grpcHandler) DescribeStream( + ctx context.Context, + req *streampb.DescribeStreamRequest, +) (*streampb.DescribeStreamResponse, error) { resp, err := g.handler.DescribeStream(ctx, req.GetFrontendRequest()) if err != nil { return nil, err diff --git a/chasm/lib/stream/handler.go b/chasm/lib/stream/handler.go index 14d980a5ad3..a5be948d855 100644 --- a/chasm/lib/stream/handler.go +++ b/chasm/lib/stream/handler.go @@ -24,13 +24,21 @@ type handler struct { historyClient resource.HistoryClient } -func newHandler(config *Config, namespaceRegistry namespace.Registry, historyClient resource.HistoryClient) *handler { - return &handler{config: config, namespaceRegistry: namespaceRegistry, historyClient: historyClient} +func newHandler( + config *Config, + namespaceRegistry namespace.Registry, + historyClient resource.HistoryClient, +) *handler { + return &handler{ + config: config, + namespaceRegistry: namespaceRegistry, + historyClient: historyClient, + } } // Reader is a Workflow read through System Nexus. type Reader struct { - // The Nexus request id, which a retry of the operation start carries again. + // The Nexus request id. A retry of the operation start sends the same id. RequestID string // The completion callback of the read's operation. A read with none answers at once. Callback *commonpb.Callback @@ -49,7 +57,11 @@ func checkProducer(producer *streamservice.Producer) error { return invalid("The Producer has no kind") } if !strings.HasPrefix(producer.GetId(), prefix) { - return invalid("The Producer id %q doesn't start with %q, as its kind requires", producer.GetId(), prefix) + return invalid( + "The Producer id %q doesn't start with %q, as its kind requires", + producer.GetId(), + prefix, + ) } return nil } @@ -95,7 +107,10 @@ func CheckStreamID(id string) error { } // target resolves the namespace, the limits and the place of a Stream. -func (h *handler) target(namespaceName, streamID string, ref *streamservice.StreamRef) (target, Limits, error) { +func (h *handler) target( + namespaceName, streamID string, + ref *streamservice.StreamRef, +) (target, Limits, error) { if err := CheckStreamID(streamID); err != nil { return target{}, Limits{}, err } @@ -175,7 +190,8 @@ func (h *handler) CreateStream( } workflowOwner := req.GetOwner().GetKind() == streamservice.PRODUCER_KIND_WORKFLOW if req.GetCloseWithRun() && (!t.workflow() || !workflowOwner) { - return nil, invalid("close_with_run needs a Workflow owner and a ref with workflow_id and first_execution_run_id") + return nil, invalid("close_with_run needs a Workflow owner and a ref with workflow_id " + + "and first_execution_run_id") } if err := h.checkChain(ctx, t, req.GetOwner(), req.GetWithdrawRun()); err != nil { return nil, err @@ -186,8 +202,11 @@ func (h *handler) CreateStream( check := func(h Host, cctx chasm.Context) error { return h.CheckOwner(cctx, req.GetOwner(), req.GetTaskToken()) } - out, err = update(ctx, t, newNestedStream, check, - func(s *Stream, mctx chasm.MutableContext) (CreateOutput, error) { return s.Create(mctx, in) }) + create := func(s *Stream, mctx chasm.MutableContext) (CreateOutput, error) { + return s.Create(mctx, in) + } + out, err = update(ctx, t, newNestedStream, check, create) + err = streamError(ctx, t, err) } else { out, err = h.createStandalone(ctx, t, in) } @@ -204,7 +223,11 @@ func (h *handler) CreateStream( }, nil } -func (h *handler) createStandalone(ctx context.Context, t target, in CreateInput) (CreateOutput, error) { +func (h *handler) createStandalone( + ctx context.Context, + t target, + in CreateInput, +) (CreateOutput, error) { create := func(policy chasm.TransitionOption) (CreateOutput, error) { result, err := chasm.UpdateWithStartExecution(ctx, t.key, StartStream, func(s *Stream, mctx chasm.MutableContext, in CreateInput) (CreateOutput, error) { @@ -276,6 +299,46 @@ func (h *handler) CloseStream( return resp, streamError(ctx, t, err) } +// CloseFromRun closes a Workflow Stream for the System Nexus command of run. Run comes from the +// link that the server's Nexus call adds, and must be an open run of the Stream's chain. The close +// uses run's Workflow Producer, and takes only the result, the failure and the status from req. +func (h *handler) CloseFromRun( + ctx context.Context, + req *streamservice.CloseStreamRequest, + run *ReaderRun, +) (*streamservice.CloseStreamResponse, error) { + t, limits, err := h.target(req.GetNamespace(), req.GetStream().GetId(), req.GetStream()) + if err != nil { + return nil, err + } + if run == nil { + return nil, notOwner("A CloseStream through System Nexus needs the link of its calling run") + } + if !t.workflow() { + return nil, notOwner("Only a Workflow Stream closes through System Nexus") + } + command := &streamservice.CloseStreamRequest{ + Namespace: req.GetNamespace(), + Stream: req.GetStream(), + Producer: &streamservice.Producer{ + Kind: streamservice.PRODUCER_KIND_WORKFLOW, + Id: "wf:" + encodePart(run.RunID), + RunId: run.RunID, + }, + Result: req.GetResult(), + Failure: req.GetFailure(), + Status: req.GetStatus(), + } + if err := h.checkChain(ctx, t, command.GetProducer(), ""); err != nil { + return nil, err + } + resp, err := update(ctx, t, nil, nil, + func(s *Stream, mctx chasm.MutableContext) (*streamservice.CloseStreamResponse, error) { + return s.Close(mctx, CloseInput{Request: command, Limits: limits, Command: true}) + }) + return resp, streamError(ctx, t, err) +} + // DescribeStream answers the state of a Stream. func (h *handler) DescribeStream( ctx context.Context, @@ -288,16 +351,24 @@ func (h *handler) DescribeStream( if err := h.applyDue(ctx, t, limits); err != nil { return nil, streamError(ctx, t, err) } - resp, err := read(ctx, t, func(s *Stream, _ chasm.Context) (*streamservice.DescribeStreamResponse, error) { - return s.Describe() - }) + resp, err := read( + ctx, + t, + func(s *Stream, _ chasm.Context) (*streamservice.DescribeStreamResponse, error) { + return s.Describe() + }, + ) return resp, streamError(ctx, t, err) } // applyDue runs a write step when a deadline such as the TTL passed, so that a read never sees // the Stream open after it expired. func (h *handler) applyDue(ctx context.Context, t target, limits Limits) error { - due, err := read(ctx, t, func(s *Stream, cctx chasm.Context) (bool, error) { return s.Due(cctx, limits) }) + due, err := read( + ctx, + t, + func(s *Stream, cctx chasm.Context) (bool, error) { return s.Due(cctx, limits) }, + ) if err != nil || !due { return err } @@ -308,9 +379,8 @@ func (h *handler) applyDue(ctx context.Context, t target, limits Limits) error { return err } -// ReadOrPark answers a Workflow read at once when the Stream has Records for it or ends. -// Otherwise it parks the read and answers nil, and the Stream completes the read's Nexus -// operation later. +// ReadOrPark answers a Workflow read at once, but parks a wake_only read with no Record for it +// and answers nil. The Stream completes the wake's Nexus operation later. func (h *handler) ReadOrPark( ctx context.Context, req *streamservice.ReadRecordsRequest, @@ -323,7 +393,13 @@ func (h *handler) ReadOrPark( if err := checkLast(req, true); err != nil { return nil, err } - in := ReadInput{Request: req, RequestID: reader.RequestID, Callback: reader.Callback, Reader: reader.Run, Limits: limits} + in := ReadInput{ + Request: req, + RequestID: reader.RequestID, + Callback: reader.Callback, + Reader: reader.Run, + Limits: limits, + } if t.nested { resp, err := update(ctx, t, newNestedStream, nil, func(s *Stream, mctx chasm.MutableContext) (*streamservice.ReadRecordsResponse, error) { @@ -331,12 +407,19 @@ func (h *handler) ReadOrPark( }) return resp, streamError(ctx, t, err) } - readStandalone := func(policy chasm.TransitionOption) (*streamservice.ReadRecordsResponse, error) { - result, err := chasm.UpdateWithStartExecution(ctx, t.key, StartForRead, - func(s *Stream, mctx chasm.MutableContext, in ReadInput) (*streamservice.ReadRecordsResponse, error) { - resp, err := s.ReadOrPark(mctx, in) - return resp, carry(err) - }, in, policy) + readOrPark := func( + s *Stream, + mctx chasm.MutableContext, + in ReadInput, + ) (*streamservice.ReadRecordsResponse, error) { + resp, err := s.ReadOrPark(mctx, in) + return resp, carry(err) + } + readStandalone := func( + policy chasm.TransitionOption, + ) (*streamservice.ReadRecordsResponse, error) { + result, err := chasm.UpdateWithStartExecution(ctx, t.key, StartForRead, readOrPark, in, + policy) return result.UpdateOutput, uncarry(err) } resp, err := readStandalone(useExisting) @@ -353,7 +436,12 @@ func (h *handler) ReadOrPark( } // CancelRead completes a parked read as canceled. -func (h *handler) CancelRead(ctx context.Context, namespaceName string, ref *streamservice.StreamRef, requestID string) error { +func (h *handler) CancelRead( + ctx context.Context, + namespaceName string, + ref *streamservice.StreamRef, + requestID string, +) error { t, limits, err := h.target(namespaceName, ref.GetId(), ref) if err != nil { return err @@ -368,12 +456,12 @@ func (h *handler) CancelRead(ctx context.Context, namespaceName string, ref *str const pollTick = 100 * time.Millisecond // longPollBuffer is the time a read leaves before the caller's deadline, so the empty answer -// gets back in time. +// reaches the caller before its deadline. const longPollBuffer = time.Second -// ReadRecords is a frontend read. It waits up to create_wait for the create and up to wait for -// a Record, and answers an empty page after the wait. A page of an open Workflow run, run, keeps -// the run's floor on a Workflow Stream first, so no trim drops what it reads. +// ReadRecords is a frontend long poll. It waits up to create_wait for the create and up to wait +// for a Record. The first page of an open Workflow run also sets the run's floor, so no trim +// drops the Records that the run read. func (h *handler) ReadRecords( ctx context.Context, req *streamservice.ReadRecordsRequest, @@ -407,7 +495,11 @@ func (h *handler) ReadRecords( }) return resp, streamError(ctx, t, err) } - start, err := read(ctx, t, func(s *Stream, _ chasm.Context) (int64, error) { return s.StartOffset(req) }) + start, err := read( + ctx, + t, + func(s *Stream, _ chasm.Context) (int64, error) { return s.StartOffset(req) }, + ) if err != nil { return nil, streamError(ctx, t, err) } @@ -436,9 +528,8 @@ func (h *handler) ReadRecords( } } -// floorRun is the run whose floor the first page keeps, or nil. That is a run whose page would -// lower its floor, mostly its first, and that is open. The link is the Worker's claim, so a run -// that is closed or doesn't exist keeps no floor. +// floorRun gives run if the first page would lower its floor and the run is open, or nil. The +// Worker sets the link, so a run that is closed or doesn't exist gets no floor. func (h *handler) floorRun( ctx context.Context, t target, @@ -446,7 +537,7 @@ func (h *handler) floorRun( from int64, run *ReaderRun, ) (*ReaderRun, error) { - if run == nil || !t.nested || req.GetWakeOnly() { + if run == nil || req.GetWakeOnly() { return nil, nil } lowers, err := read(ctx, t, func(s *Stream, cctx chasm.Context) (bool, error) { @@ -463,9 +554,8 @@ func (h *handler) floorRun( return run, nil } -// page reads one page from from. With floorRun set, the transaction that answers the page keeps -// the run's floor too, so no trim comes between the answer and the floor. The page of a reader -// run suggests continue-as-new when the run's floor keeps too much. +// page reads one page at from. With floorRun set, the transaction that answers the page also keeps +// the run's floor, so no trim runs between the answer and the floor. func (h *handler) page( ctx context.Context, t target, @@ -528,7 +618,12 @@ func (h *handler) waitForCreate(ctx context.Context, t target, until time.Time) // waitForChange waits until the Stream has an entry at or past cursor, or is no longer open, or // the deadline passes. The expiry is a timer task, so it ends the wait too. -func (h *handler) waitForChange(ctx context.Context, t target, cursor int64, deadline time.Time) error { +func (h *handler) waitForChange( + ctx context.Context, + t target, + cursor int64, + deadline time.Time, +) error { pollCtx, cancel := context.WithDeadline(ctx, deadline) defer cancel() cursor = max(cursor, 1) diff --git a/chasm/lib/stream/host.go b/chasm/lib/stream/host.go index b8ec5b1b14c..1a1f7a450cb 100644 --- a/chasm/lib/stream/host.go +++ b/chasm/lib/stream/host.go @@ -17,16 +17,16 @@ import ( "go.temporal.io/server/common/nexus/nexusrpc" ) -// Host is a component that nests Streams: the Workflow, for the Streams of its run chain, or a -// Standalone Activity, for its own. A nested Stream commits in its host's transaction, so an -// append and the wake of its readers go together. +// Host is a component that nests Streams. A Workflow hosts the Streams of its run chain, and a +// Standalone Activity hosts its own. A nested Stream commits in its host's transaction, so an +// append and the wake of its readers commit together. type Host interface { chasm.Component NestedStreams() chasm.Map[string, *Stream] - // CheckOwner refuses a create whose owner the host doesn't prove, with the task token of its - // current attempt. A Workflow's chain is checked before the transaction. + // CheckOwner refuses a create whose owner isn't the host. A Standalone Activity checks the + // task token of its current attempt. CheckOwner(ctx chasm.Context, owner *streamservice.Producer, token []byte) error - // CheckWriter refuses a write that claims the host's own Producer with no proof. + // CheckWriter refuses a write as the host's own Producer from any other caller. CheckWriter(ctx chasm.Context, producer *streamservice.Producer, token []byte) error } @@ -37,8 +37,8 @@ type ChainHost interface { RunEnd(ctx chasm.Context) (RunEnd, error) } -// ActivityArchetypeID is the archetype of the Activity library's component, which hosts a -// Standalone Activity's Streams. The activity library imports this one, so the id is made here. +// ActivityArchetypeID is the archetype of the Activity library's component. The activity library +// imports this package, so this package can't take the id from there. var ActivityArchetypeID = chasm.GenerateTypeID(chasm.FullyQualifiedName("activity", "activity")) // RunEnd is how the run that ends a chain ended. The chain's close-with-run Streams close with it. @@ -64,8 +64,20 @@ func (s *Stream) EndChain(ctx chasm.MutableContext, end RunEnd, limits Limits) { s.finish(ctx, limits) } +// HoldsStreams says whether a Stream of a closed host is still readable, so the host must outlive +// the namespace retention that would delete it. +func HoldsStreams(ctx chasm.Context, streams chasm.Map[string, *Stream]) bool { + for _, field := range streams { + if s, ok := field.TryGet(ctx); ok && s.Phase != streampb.STREAM_PHASE_EXPIRED && + s.Phase != streampb.STREAM_PHASE_ABANDONED { + return true + } + } + return false +} + // EndOwner closes a Standalone Activity's Stream with the Activity's end, in the transition that -// ends it. A retry doesn't end the Activity, so the Stream goes on across attempts. +// ends it. A retry doesn't end the Activity, so the Stream stays open across attempts. func (s *Stream) EndOwner(ctx chasm.MutableContext, end RunEnd, limits Limits) { if s.Phase == streampb.STREAM_PHASE_OPEN { s.close(ctx, s.GetOwner(), end.Result, end.Failure, end.Status) @@ -73,8 +85,8 @@ func (s *Stream) EndOwner(ctx chasm.MutableContext, end RunEnd, limits Limits) { s.finish(ctx, limits) } -// target is where a Stream lives: its own execution keyed by the Stream id, or a child of its -// host, the Workflow component of its chain's first run or a Standalone Activity. +// target is where a Stream lives. That's its own execution keyed by the Stream id, or a child of +// its host, which is the Workflow component of its chain's first run or a Standalone Activity. type target struct { key chasm.ExecutionKey id string @@ -93,7 +105,7 @@ func (t target) activity() bool { } // targetOf finds the Stream that a ref names. A Workflow Stream nests in the first run of its -// chain, since a run's CHASM tree doesn't move to the next run. +// chain, since each run of a chain has its own CHASM tree. func targetOf(namespaceID, streamID string, ref *streamservice.StreamRef) (target, error) { if ref.GetWorkflowId() != "" && ref.GetActivityId() != "" { return target{}, invalid("A Stream ref has a Workflow or an Activity, not both") @@ -112,7 +124,10 @@ func targetOf(namespaceID, streamID string, ref *streamservice.StreamRef) (targe }, nil } if ref.GetWorkflowId() == "" { - return target{key: chasm.ExecutionKey{NamespaceID: namespaceID, BusinessID: streamID}, id: streamID}, nil + return target{ + key: chasm.ExecutionKey{NamespaceID: namespaceID, BusinessID: streamID}, + id: streamID, + }, nil } if ref.GetFirstExecutionRunId() == "" { return target{}, invalid("A Workflow Stream ref needs first_execution_run_id") @@ -129,7 +144,8 @@ func targetOf(namespaceID, streamID string, ref *streamservice.StreamRef) (targe }, nil } -// newNestedStream makes a nested Stream, whose Workflow readers keep their floors. +// newNestedStream makes a nested Stream. Its reads see every Record it stores, also past the +// Retention. func newNestedStream(ctx chasm.MutableContext) (*Stream, error) { s := NewStream(ctx, streampb.STREAM_PHASE_PENDING) s.Nested = true @@ -155,7 +171,9 @@ func update[O any]( } if !t.nested { out, _, err := chasm.UpdateComponent(ctx, chasm.NewComponentRef[*Stream](t.key), - func(s *Stream, mctx chasm.MutableContext, _ struct{}) (O, error) { return fn(s, mctx) }, struct{}{}) + func(s *Stream, mctx chasm.MutableContext, _ struct{}) (O, error) { + return fn(s, mctx) + }, struct{}{}) return out, uncarry(err) } out, _, err := chasm.UpdateComponent(ctx, t.hostRef(), @@ -181,21 +199,31 @@ func update[O any]( if err != nil { return zero, err } - streams[t.id] = chasm.NewComponentField(mctx, s) + // A Stream outlives its host's close, as in a chain's first run or an ended Activity, + // so its timers, wakes and reader checks must run under a closed host. + streams[t.id] = chasm.NewComponentField(mctx, s, chasm.ComponentFieldDetached()) return out, nil }, struct{}{}) return out, uncarry(err) } // read runs fn on the Stream with no write. -func read[O any](ctx context.Context, t target, view func(*Stream, chasm.Context) (O, error)) (O, error) { +func read[O any]( + ctx context.Context, + t target, + view func(*Stream, chasm.Context) (O, error), +) (O, error) { fn := func(s *Stream, cctx chasm.Context) (O, error) { out, err := view(s, cctx) return out, carry(err) } if !t.nested { - out, err := chasm.ReadComponent(ctx, chasm.NewComponentRef[*Stream](t.key), - func(s *Stream, cctx chasm.Context, _ struct{}) (O, error) { return fn(s, cctx) }, struct{}{}) + out, err := chasm.ReadComponent( + ctx, + chasm.NewComponentRef[*Stream](t.key), + func(s *Stream, cctx chasm.Context, _ struct{}) (O, error) { return fn(s, cctx) }, + struct{}{}, + ) return out, uncarry(err) } out, err := chasm.ReadComponent(ctx, t.hostRef(), @@ -234,9 +262,14 @@ func uncarry(err error) error { func poll(ctx context.Context, t target, done func(*Stream) bool) error { var err error if !t.nested { - _, _, err = chasm.PollComponent(ctx, chasm.NewComponentRef[*Stream](t.key), - func(s *Stream, _ chasm.Context, _ struct{}) (struct{}, bool, error) { return struct{}{}, done(s), nil }, - struct{}{}) + _, _, err = chasm.PollComponent( + ctx, + chasm.NewComponentRef[*Stream](t.key), + func(s *Stream, _ chasm.Context, _ struct{}) (struct{}, bool, error) { + return struct{}{}, done(s), nil + }, + struct{}{}, + ) } else { _, _, err = chasm.PollComponent(ctx, t.hostRef(), func(h Host, cctx chasm.Context, _ struct{}) (struct{}, bool, error) { @@ -256,7 +289,10 @@ type closeChainTaskHandler struct { namespaceRegistry namespace.Registry } -func newCloseChainTaskHandler(config *Config, namespaceRegistry namespace.Registry) *closeChainTaskHandler { +func newCloseChainTaskHandler( + config *Config, + namespaceRegistry namespace.Registry, +) *closeChainTaskHandler { return &closeChainTaskHandler{config: config, namespaceRegistry: namespaceRegistry} } @@ -288,10 +324,26 @@ func (h *closeChainTaskHandler) Execute( } limits := h.config.LimitsFor(ns.Name().String()) first := target{ - key: chasm.ExecutionKey{NamespaceID: ref.NamespaceID, BusinessID: ref.BusinessID, RunID: task.GetFirstRunId()}, + key: chasm.ExecutionKey{ + NamespaceID: ref.NamespaceID, + BusinessID: ref.BusinessID, + RunID: task.GetFirstRunId(), + }, nested: true, host: chasm.WorkflowArchetypeID, } + // A later run can't tell whether the first run has Streams, so every chain end comes here. A + // read first spares the first run a write when it has none. + nests, err := chasm.ReadComponent(ctx, first.hostRef(), + func(h Host, _ chasm.Context, _ struct{}) (bool, error) { + return len(h.NestedStreams()) > 0, nil + }, struct{}{}) + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { + return nil + } + if err != nil || !nests { + return err + } _, _, err = chasm.UpdateComponent(ctx, first.hostRef(), func(h Host, mctx chasm.MutableContext, _ struct{}) (struct{}, error) { for _, field := range h.NestedStreams() { diff --git a/chasm/lib/stream/library.go b/chasm/lib/stream/library.go index c9f79326a72..1c98af93762 100644 --- a/chasm/lib/stream/library.go +++ b/chasm/lib/stream/library.go @@ -36,7 +36,14 @@ func newLibrary( chain *closeChainTaskHandler, readers *readersCheckTaskHandler, ) *library { - return &library{config: config, handler: handler, grpc: grpcHandler, timer: timer, chain: chain, readers: readers} + return &library{ + config: config, + handler: handler, + grpc: grpcHandler, + timer: timer, + chain: chain, + readers: readers, + } } // NewNilLibrary creates a Library with nil handlers, for readers that only decode a Stream. diff --git a/chasm/lib/stream/nexus.go b/chasm/lib/stream/nexus.go index f40af5d22a8..f6728b67a0d 100644 --- a/chasm/lib/stream/nexus.go +++ b/chasm/lib/stream/nexus.go @@ -22,17 +22,18 @@ const ServiceName = "temporal.api.streamservice.v1.StreamService" func readerOf(options nexus.StartOperationOptions) Reader { reader := Reader{RequestID: options.RequestID, Run: readerRunOf(options.Links)} if options.CallbackURL != "" { - reader.Callback = &commonpb.Callback{Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{ - Url: options.CallbackURL, - Header: options.CallbackHeader, - }}} + reader.Callback = &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{ + Url: options.CallbackURL, + Header: options.CallbackHeader, + }}, + } } return reader } -// nexusError gives the Nexus form of a Stream error. A Stream error fails the operation with -// an application failure whose type is the reason token, so the Workflow sees the same error -// class as a client. Any other error stays a handler error, which the caller retries. +// nexusError converts a Stream error to an application failure whose type is the reason token, so +// a Workflow sees the same error class as a client. Other errors stay handler errors for retry. func nexusError(err error) error { if err == nil { return nil @@ -50,26 +51,51 @@ func nexusError(err error) error { return err } -// newNexusService serves the reads to Workflow code. Workflow writes go over the frontend from -// the Core extension, which knows the Workflow Task facts of each write. +// newNexusService serves the reads and the close to Workflow code. The close is a command, so the +// server runs it only when the Workflow Task that issued it commits. func newNexusService(h *handler) *nexus.Service { service := nexus.NewService(ServiceName) service.MustRegister( - nexus.NewSyncOperation("DescribeStream", - func(ctx context.Context, req *streamservice.DescribeStreamRequest, _ nexus.StartOperationOptions) (*streamservice.DescribeStreamResponse, error) { + nexus.NewSyncOperation( + "DescribeStream", + func( + ctx context.Context, + req *streamservice.DescribeStreamRequest, + _ nexus.StartOperationOptions, + ) (*streamservice.DescribeStreamResponse, error) { resp, err := h.DescribeStream(ctx, req) return resp, nexusError(err) - }), + }, + ), + nexus.NewSyncOperation( + "CloseStream", + func( + ctx context.Context, + req *streamservice.CloseStreamRequest, + options nexus.StartOperationOptions, + ) (*streamservice.CloseStreamResponse, error) { + // A command's start always has the system callback, and a direct call never has + // one. A direct call waits for no Workflow Task to commit, so it can't close. + if options.CallbackURL == "" { + return nil, nexusError(invalid("A direct System Nexus call can't close a " + + "Stream, so use the gRPC CloseStream")) + } + resp, err := h.CloseFromRun(ctx, req, readerRunOf(options.Links)) + return resp, nexusError(err) + }, + ), &readRecordsOperation{handler: h}, ) return service } -// readRecordsOperation answers a read at once when the Stream has Records for it or ends. Any -// other read is an async operation that the Stream completes when Records come or the Stream -// closes, so a Workflow waits on the server with no timer. +// readRecordsOperation answers a read at once, but a wake_only read with no Record is async. The +// Stream completes it on a Record or the close, so a Workflow waits on the server with no timer. type readRecordsOperation struct { - nexus.UnimplementedOperation[*streamservice.ReadRecordsRequest, *streamservice.ReadRecordsResponse] + nexus.UnimplementedOperation[ + *streamservice.ReadRecordsRequest, + *streamservice.ReadRecordsResponse, + ] handler *handler } @@ -94,7 +120,9 @@ func (o *readRecordsOperation) Start( if err != nil { return nil, nexusError(err) } - return &nexus.HandlerStartOperationResultSync[*streamservice.ReadRecordsResponse]{Value: resp}, nil + return &nexus.HandlerStartOperationResultSync[*streamservice.ReadRecordsResponse]{ + Value: resp, + }, nil } resp, err := o.handler.ReadOrPark(ctx, req, readerOf(options)) if err != nil { @@ -115,10 +143,16 @@ func (o *readRecordsOperation) Start( } return &nexus.HandlerStartOperationResultAsync{OperationToken: string(token)}, nil } - return &nexus.HandlerStartOperationResultSync[*streamservice.ReadRecordsResponse]{Value: resp}, nil + return &nexus.HandlerStartOperationResultSync[*streamservice.ReadRecordsResponse]{ + Value: resp, + }, nil } -func (o *readRecordsOperation) Cancel(ctx context.Context, token string, _ nexus.CancelOperationOptions) error { +func (o *readRecordsOperation) Cancel( + ctx context.Context, + token string, + _ nexus.CancelOperationOptions, +) error { var t readToken if err := json.Unmarshal([]byte(token), &t); err != nil { return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation token") @@ -195,7 +229,11 @@ func (p processor[R]) ProcessInput( p.setNamespace(req, name) case name: default: - return nil, serviceerror.NewInvalidArgumentf("Namespace in request %q does not match namespace in context %q", req.GetNamespace(), name) + return nil, serviceerror.NewInvalidArgumentf( + "Namespace in request %q does not match namespace in context %q", + req.GetNamespace(), + name, + ) } businessID := ref.GetWorkflowId() if businessID == "" { @@ -214,13 +252,30 @@ func (p processor[R]) ProcessInput( func newNexusServiceProcessor(config *Config) *chasm.NexusServiceProcessor { sp := chasm.NewNexusServiceProcessor(ServiceName) - sp.MustRegisterOperation("ReadRecords", chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.ReadRecordsRequest]{ - config: config, - setNamespace: func(r *streamservice.ReadRecordsRequest, ns string) { r.Namespace = ns }, - })) - sp.MustRegisterOperation("DescribeStream", chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.DescribeStreamRequest]{ - config: config, - setNamespace: func(r *streamservice.DescribeStreamRequest, ns string) { r.Namespace = ns }, - })) + sp.MustRegisterOperation( + "ReadRecords", + chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.ReadRecordsRequest]{ + config: config, + setNamespace: func(r *streamservice.ReadRecordsRequest, ns string) { r.Namespace = ns }, + }), + ) + sp.MustRegisterOperation( + "DescribeStream", + chasm.NewRegisterableNexusOperationProcessor( + processor[*streamservice.DescribeStreamRequest]{ + config: config, + setNamespace: func(r *streamservice.DescribeStreamRequest, ns string) { + r.Namespace = ns + }, + }, + ), + ) + sp.MustRegisterOperation( + "CloseStream", + chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.CloseStreamRequest]{ + config: config, + setNamespace: func(r *streamservice.CloseStreamRequest, ns string) { r.Namespace = ns }, + }), + ) return sp } diff --git a/chasm/lib/stream/nexus_test.go b/chasm/lib/stream/nexus_test.go index a7be7863182..4309f9cb36c 100644 --- a/chasm/lib/stream/nexus_test.go +++ b/chasm/lib/stream/nexus_test.go @@ -23,7 +23,9 @@ func TestNexusErrorFailsTheOperationWithTheReason(t *testing.T) { require.True(t, ok) require.Equal(t, nexus.OperationStateFailed, opErr.State) - _, ok = nexusError(serviceerror.NewResourceExhausted(enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, "full")).(*nexus.HandlerError) + full := serviceerror.NewResourceExhausted( + enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, "full") + _, ok = nexusError(full).(*nexus.HandlerError) require.True(t, ok) unavailable := serviceerror.NewUnavailable("down") @@ -32,25 +34,72 @@ func TestNexusErrorFailsTheOperationWithTheReason(t *testing.T) { func TestProcessorRoutesByStreamID(t *testing.T) { ns := namespace.NewLocalNamespaceForTest( - &persistencespb.NamespaceInfo{Id: "ns-id", Name: "ns"}, &persistencespb.NamespaceConfig{}, "active") + &persistencespb.NamespaceInfo{ + Id: "ns-id", + Name: "ns", + }, + &persistencespb.NamespaceConfig{}, + "active", + ) collection := dynamicconfig.NewMemoryClient() collection.OverrideSetting(Enabled, true) - sp := newNexusServiceProcessor(ConfigProvider(dynamicconfig.NewCollection(collection, log.NewNoopLogger()))) + sp := newNexusServiceProcessor( + ConfigProvider(dynamicconfig.NewCollection(collection, log.NewNoopLogger())), + ) ctx := chasm.NexusOperationProcessorContext{Namespace: ns, ReserializeInputPayload: true} - input := payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{Stream: &streamservice.StreamRef{Id: "chat-1"}}) + input := payloads.MustEncodeSingle( + &streamservice.ReadRecordsRequest{Stream: &streamservice.StreamRef{Id: "chat-1"}}, + ) result, err := sp.ProcessInput(ctx, "ReadRecords", input) require.NoError(t, err) - require.Equal(t, chasm.NexusOperationRoutingKeyExecution{NamespaceID: "ns-id", BusinessID: "chat-1"}, result.RoutingKey) + require.Equal( + t, + chasm.NexusOperationRoutingKeyExecution{NamespaceID: "ns-id", BusinessID: "chat-1"}, + result.RoutingKey, + ) var routed streamservice.ReadRecordsRequest - require.NoError(t, payloads.Decode(&commonpb.Payloads{Payloads: []*commonpb.Payload{result.ReserializedInputPayload}}, &routed)) + require.NoError( + t, + payloads.Decode( + &commonpb.Payloads{Payloads: []*commonpb.Payload{result.ReserializedInputPayload}}, + &routed, + ), + ) require.Equal(t, "ns", routed.GetNamespace()) - _, err = sp.ProcessInput(ctx, "ReadRecords", payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{})) + // A Workflow Stream is on its Workflow's shard. + result, err = sp.ProcessInput( + ctx, + "CloseStream", + payloads.MustEncodeSingle(&streamservice.CloseStreamRequest{ + Stream: &streamservice.StreamRef{ + Id: "chat-1", + WorkflowId: "wf", + FirstExecutionRunId: "first", + }, + }), + ) + require.NoError(t, err) + require.Equal( + t, + chasm.NexusOperationRoutingKeyExecution{NamespaceID: "ns-id", BusinessID: "wf"}, + result.RoutingKey, + ) + + _, err = sp.ProcessInput( + ctx, + "ReadRecords", + payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{}), + ) require.ErrorContains(t, err, ReasonInvalid) - _, err = sp.ProcessInput(ctx, "ReadRecords", payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{ - Namespace: "other", - Stream: &streamservice.StreamRef{Id: "chat-1"}, - })) + _, err = sp.ProcessInput( + ctx, + "ReadRecords", + payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{ + Namespace: "other", + Stream: &streamservice.StreamRef{Id: "chat-1"}, + }), + ) require.ErrorContains(t, err, "does not match") } diff --git a/chasm/lib/stream/owner.go b/chasm/lib/stream/owner.go index d9769535a38..75218b2f463 100644 --- a/chasm/lib/stream/owner.go +++ b/chasm/lib/stream/owner.go @@ -14,8 +14,7 @@ import ( "go.temporal.io/server/common/tasktoken" ) -// The checks of a Stream's owner and of the Producers that claim a Workflow run or an Activity -// attempt. A caller with write access could otherwise close any Stream, write as a Workflow, +// Without these checks, a caller with write access could close any Stream, write as a Workflow, // fence a run, or put a Stream into a Workflow's first run, which then outlives its retention. func notOwner(format string, args ...any) error { @@ -30,9 +29,8 @@ func ownerKind(owner *streamservice.Producer) streamservice.ProducerKind { return streamservice.PRODUCER_KIND_CLIENT } -// checkOwner refuses a create whose owner can't own a Stream. A Workflow Stream nests in its -// chain's first run, so only that chain creates one. A Standalone Activity's Stream nests in the -// Activity, so only the Activity creates one, and its host checks the task token. +// checkOwner refuses a create whose owner can't own a Stream. A nested Stream is a child of its +// host, so only the host creates it. An Activity host checks the task token as well. func checkOwner(owner *streamservice.Producer, t target) error { switch kind := ownerKind(owner); { case t.workflow() && kind != streamservice.PRODUCER_KIND_WORKFLOW: @@ -41,6 +39,9 @@ func checkOwner(owner *streamservice.Producer, t target) error { return notOwner("Only the Standalone Activity creates its Stream") case kind == streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY && !t.activity(): return invalid("A Standalone Activity owns only a Stream in its ref") + // Outside a chain the Stream would get the safety TTL with no chain check and no close. + case kind == streamservice.PRODUCER_KIND_WORKFLOW && !t.workflow(): + return invalid("A Workflow owns only a Stream in its ref") case kind == streamservice.PRODUCER_KIND_WORKFLOW || kind == streamservice.PRODUCER_KIND_CLIENT || kind == streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY: @@ -51,15 +52,16 @@ func checkOwner(owner *streamservice.Producer, t target) error { } } -// checkCloser refuses a close that the Stream's owner doesn't allow. A Workflow Stream closes -// only from its run chain, and a client's Stream only from a client. The server's own close with -// the run chain comes from no caller. +// checkCloser refuses a close by a Producer of another kind than the Stream's owner. The close +// with the run chain comes from the server, so it doesn't go through this check. func (s *Stream) checkCloser(producer *streamservice.Producer) error { kind := producer.GetKind() owner := ownerKind(s.GetOwner()) switch owner { case streamservice.PRODUCER_KIND_WORKFLOW: - if ofRun(kind) { + // A Local Activity writes before its Workflow Task commits, and a failed task can't + // take a close back, so only the run's close Activity or command closes. + if kind == streamservice.PRODUCER_KIND_WORKFLOW { return nil } case streamservice.PRODUCER_KIND_CLIENT: @@ -83,8 +85,8 @@ func ofRun(kind streamservice.ProducerKind) bool { kind == streamservice.PRODUCER_KIND_LOCAL_ACTIVITY } -// checkRunID refuses a Producer whose id doesn't carry its run, so a write can't claim a run in -// one field and show another in its id. +// checkRunID refuses a Producer whose id doesn't contain its run id, so a write can't claim one +// run in run_id and another in its id. func checkRunID(producer *streamservice.Producer) error { run := encodePart(producer.GetRunId()) id := producer.GetId() @@ -124,7 +126,7 @@ func (h *handler) chainRun(ctx context.Context, t target, runID string) (bool, e resp.GetWorkflowStatus() == enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, nil } -// nextRun says whether next is the run that follows run in the chain, so only a retry run +// nextRun says whether next is the run after run in the chain, so only a retry run // withdraws its previous run. func (h *handler) nextRun(ctx context.Context, t target, run, next string) (bool, error) { req := &historyservice.DescribeMutableStateRequest{ @@ -182,8 +184,7 @@ func (h *handler) checkChain( } // checkActivity refuses an Activity's write whose task token isn't the Activity's current -// attempt. The epoch stays the Producer's, (attempt scheduled time, attempt), and its attempt -// must be the token's. +// attempt. The Producer's run, id and epoch attempt must match the token. func (h *handler) checkActivity( ctx context.Context, namespaceName string, diff --git a/chasm/lib/stream/parked_read.go b/chasm/lib/stream/parked_read.go index 71d9d0ad1d4..9a8cda76e23 100644 --- a/chasm/lib/stream/parked_read.go +++ b/chasm/lib/stream/parked_read.go @@ -17,8 +17,8 @@ import ( var errCanceled = errors.New("the read was canceled") -// ParkedRead is a Workflow read that found no Record. It holds the completion callback of the -// read's Nexus operation, and the Stream completes it when the read can answer. +// ParkedRead is a Workflow's wake_only read that found no Record. It holds the completion +// callback of the read's Nexus operation, and the Stream completes it when the read can answer. type ParkedRead struct { chasm.UnimplementedComponent @@ -52,7 +52,10 @@ func (r *ParkedRead) CompletionCallbackMetadata() *callbackspb.CallbackMetadata // GetNexusCompletion gives the page as the result, or the error as the failure of the // operation. -func (r *ParkedRead) GetNexusCompletion(ctx chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { +func (r *ParkedRead) GetNexusCompletion( + ctx chasm.Context, + _ string, +) (nexusrpc.CompleteOperationOptions, error) { opts := nexusrpc.CompleteOperationOptions{StartTime: r.GetStartTime().AsTime()} if r.GetFailure() == nil { result, err := SystemPayload(r.GetResponse()) @@ -70,7 +73,11 @@ func (r *ParkedRead) GetNexusCompletion(ctx chasm.Context, _ string) (nexusrpc.C return opts, nil } -func (r *ParkedRead) resolve(ctx chasm.MutableContext, resp *streamservice.ReadRecordsResponse, err error) { +func (r *ParkedRead) resolve( + ctx chasm.MutableContext, + resp *streamservice.ReadRecordsResponse, + err error, +) { r.Resolved = true r.CreateDeadline = nil if err != nil { @@ -88,19 +95,25 @@ func (r *ParkedRead) resolve(ctx chasm.MutableContext, resp *streamservice.ReadR func FailureOf(err error) *failurepb.Failure { if errors.Is(err, errCanceled) { return &failurepb.Failure{ - Message: "The read was canceled", - FailureInfo: &failurepb.Failure_CanceledFailureInfo{CanceledFailureInfo: &failurepb.CanceledFailureInfo{}}, + Message: "The read was canceled", + FailureInfo: &failurepb.Failure_CanceledFailureInfo{ + CanceledFailureInfo: &failurepb.CanceledFailureInfo{}, + }, } } reason := ReasonOf(err) - details, _ := sdkconverter.PreferProtoDataConverter.ToPayloads(&streamservice.StreamErrorDetails{Reason: reason}) + details, _ := sdkconverter.PreferProtoDataConverter.ToPayloads( + &streamservice.StreamErrorDetails{Reason: reason}, + ) return &failurepb.Failure{ Message: err.Error(), - FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ - Type: reason, - NonRetryable: true, - Details: details, - }}, + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: reason, + NonRetryable: true, + Details: details, + }, + }, } } diff --git a/chasm/lib/stream/producer.go b/chasm/lib/stream/producer.go index 3abf10f4def..03845d04c77 100644 --- a/chasm/lib/stream/producer.go +++ b/chasm/lib/stream/producer.go @@ -34,22 +34,33 @@ func (s *Stream) admit( } known := s.producer(ctx, producerKey(producer)) if known != nil && CompareEpochs(producer.GetEpoch(), known.GetEpoch()) < 0 { - return false, NewError(ReasonFenced, "The epoch is lower than the newest one of the Producer") + return false, NewError( + ReasonFenced, + "The epoch is lower than the newest one of the Producer", + ) } run := s.run(ctx, producer.GetRunId()) task := taskOf(producer.GetEpoch()) switch producer.GetKind() { case streamservice.PRODUCER_KIND_LOCAL_ACTIVITY: - if run != nil && (slices.ContainsFunc(run.GetWithdrawn(), func(t *streamservice.WorkflowTaskEpoch) bool { - return compareTasks(t, task) == 0 - }) || (run.GetLastTask() != nil && len(withdrawTasks( - []*streamservice.WorkflowTaskEpoch{task}, run.GetLastTask(), run.GetLastCompleted(), run.GetLastFailed(), - ).withdrawn) > 0)) { - return false, NewError(ReasonFenced, "A Superseded Record withdrew the Workflow Task of the call") + withdrawn := slices.ContainsFunc(run.GetWithdrawn(), + func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }) + if run.GetLastTask() != nil { + index := []*streamservice.WorkflowTaskEpoch{task} + split := withdrawTasks(index, run.GetLastTask(), run.GetLastCompleted(), + run.GetLastFailed()) + withdrawn = withdrawn || len(split.withdrawn) > 0 + } + if withdrawn { + return false, NewError( + ReasonFenced, + "A Superseded Record withdrew the Workflow Task of the call", + ) } return newEpoch(producer, known) || s.withdrawsRun(ctx, w.WithdrawRun), nil case streamservice.PRODUCER_KIND_WORKFLOW: - flagged := !w.Defer && len(withdrawTasks(run.GetIndex(), task, w.PreviousCompleted, w.FailedTasks).withdrawn) > 0 + split := withdrawTasks(run.GetIndex(), task, w.PreviousCompleted, w.FailedTasks) + flagged := !w.Defer && len(split.withdrawn) > 0 return flagged || s.withdrawsRun(ctx, w.WithdrawRun), nil default: return newEpoch(producer, known) || s.withdrawsRun(ctx, w.WithdrawRun), nil @@ -57,14 +68,13 @@ func (s *Stream) admit( } // newEpoch says whether a write is the first of a higher epoch of a Producer whose lower epoch -// wrote Records, so its Superseded Record withdraws them, as the crate's append script does. A -// Local Activity's next attempt in the same Workflow Task redoes the call. +// wrote Records, so its Superseded Record withdraws them. A Local Activity's retry redoes the call. func newEpoch(producer *streamservice.Producer, known *streampb.ProducerState) bool { return withdrawsEpochs(producer.GetKind()) && known != nil && known.GetHolds() != nil && CompareEpochs(known.GetHolds(), producer.GetEpoch()) < 0 } -// settle keeps the facts of a write that went in: the epoch, the task index and the run. +// settle keeps the task facts and the fenced run of a write that succeeded. func (s *Stream) settle( ctx chasm.MutableContext, producer *streamservice.Producer, @@ -88,14 +98,17 @@ func (s *Stream) settle( } if wroteRecords { run.Wrote = true - if !slices.ContainsFunc(run.Index, func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }) { + if !slices.ContainsFunc( + run.Index, + func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }, + ) { run.Index = append(run.Index, task) } } } -// withdrawn says whether the Stream knows that a Superseded Record withdrew the Record: its run -// is fenced, its Workflow Task was withdrawn, or a higher epoch of its Producer wrote. +// withdrawn says whether the Stream knows that a Superseded Record withdrew the Record. It knows +// when the run is fenced, the Workflow Task was withdrawn, or a higher epoch of the Producer wrote. func (s *Stream) withdrawn(ctx chasm.Context, record *streamservice.StreamRecord) bool { producer := record.GetProducer() if s.runFenced(producer.GetRunId()) { @@ -115,9 +128,8 @@ func (s *Stream) withdrawn(ctx chasm.Context, record *streamservice.StreamRecord CompareEpochs(producer.GetEpoch(), known.GetEpoch()) < 0 } -// withdrawsEpochs says whether a higher epoch of a Producer outside Workflow code withdraws the -// Records of its lower epochs. An Activity's retry redoes the attempt's work. A client's higher -// epoch only fences the lower ones, since a client process writes on and doesn't redo. +// withdrawsEpochs says whether a non-Workflow Producer's higher epoch withdraws its lower epochs' +// Records. An Activity's retry redoes the attempt's work, but a client doesn't redo its writes. func withdrawsEpochs(kind streamservice.ProducerKind) bool { return kind != streamservice.PRODUCER_KIND_WORKFLOW && kind != streamservice.PRODUCER_KIND_CLIENT @@ -136,7 +148,8 @@ func (s *Stream) withdrawsRun(ctx chasm.Context, run string) bool { func supersededRecord(producer *streamservice.Producer, w Withdrawal) *streamservice.StreamRecord { epoch := producer.GetEpoch() if w.Defer { - // A deferred batch withdraws only the retry run's previous run, so no epoch goes with it. + // A deferred batch or a create withdraws only the retry run's previous run, so its + // Superseded Record has no epoch. epoch = nil } return &streamservice.StreamRecord{ @@ -153,7 +166,7 @@ func supersededRecord(producer *streamservice.Producer, w Withdrawal) *streamser } } -// orphans applies to a Local Activity write with its run's last completed task (S110). It splits +// orphans applies to a Local Activity write that has its run's last completed task. It splits // the run's open Local Activity tasks into those that never completed and those still open. func (s *Stream) orphans( ctx chasm.Context, @@ -179,7 +192,11 @@ func (s *Stream) settleOrphans( run := s.runForWrite(ctx, producer.GetRunId()) run.OpenLocal = split.kept task := taskOf(producer.GetEpoch()) - if wroteRecords && !slices.ContainsFunc(run.OpenLocal, func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }) { + if wroteRecords && + !slices.ContainsFunc( + run.OpenLocal, + func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }, + ) { run.OpenLocal = append(run.OpenLocal, task) } run.Withdrawn = append(run.Withdrawn, split.withdrawn...) @@ -224,9 +241,8 @@ func completedTask(req *streamservice.AppendRecordsRequest) *streamservice.Workf return nil } -// orphanRecord withdraws a run's orphaned Local Activity Records. It names the run's Workflow -// Producer in the write's task, so a reader's Workflow rule withdraws the run's Local Activity -// Records of the tasks after the completed one and before that task, and of the failed ones. +// orphanRecord is a Superseded Record of the run's Workflow Producer at the write's task, so a +// reader's Workflow rule withdraws the run's orphaned Local Activity Records. func orphanRecord(req *streamservice.AppendRecordsRequest) *streamservice.StreamRecord { producer := req.GetProducer() epoch := producer.GetEpoch() @@ -247,7 +263,7 @@ func orphanRecord(req *streamservice.AppendRecordsRequest) *streamservice.Stream } } -// EncodeIDPart percent-encodes an id part as the crate does, so a Producer id matches its. +// EncodeIDPart percent-encodes an id part as the crate does, so a Producer id matches the crate's. func EncodeIDPart(text string) string { return encodePart(text) } @@ -270,10 +286,8 @@ type taskSplit struct { kept []*streamservice.WorkflowTaskEpoch } -// withdrawTasks is the crate's Workflow withdrawal rule over a task index. For task t with the -// previous completed task c, it withdraws the tasks in failed or strictly between c and t, and -// those at the started id of c or of t with a lower attempt. It keeps the tasks after c, since -// no task up to c can be withdrawn later. +// withdrawTasks is the crate's Workflow withdrawal rule for task t and the previous completed task +// c. The index keeps only the tasks after c, since no later write can withdraw a task up to c. func withdrawTasks( index []*streamservice.WorkflowTaskEpoch, t, c *streamservice.WorkflowTaskEpoch, diff --git a/chasm/lib/stream/proto/v1/message.proto b/chasm/lib/stream/proto/v1/message.proto index 17d5545c9f3..a9a2e937a37 100644 --- a/chasm/lib/stream/proto/v1/message.proto +++ b/chasm/lib/stream/proto/v1/message.proto @@ -129,7 +129,7 @@ message ParkedReadState { temporal.api.streamservice.v1.ReadRecordsRequest request = 1; // The Nexus request id, so a retry of the start finds this read. string request_id = 2; - // Where the next look starts. A look that finds only other Topics moves it on. + // Where the next look starts. A look that finds only other Topics advances it. int64 scan_offset = 3; // Set while the read waits for the create. google.protobuf.Timestamp create_deadline = 4; diff --git a/chasm/lib/stream/proto/v1/service.proto b/chasm/lib/stream/proto/v1/service.proto index 2ab7e88d358..3849baf0933 100644 --- a/chasm/lib/stream/proto/v1/service.proto +++ b/chasm/lib/stream/proto/v1/service.proto @@ -8,7 +8,7 @@ import "temporal/server/api/routing/v1/extension.proto"; option go_package = "go.temporal.io/server/chasm/lib/stream/gen/streampb;streampb"; -// StreamService carries the frontend's Stream calls to the history shard of the Stream. A Workflow +// StreamService serves the frontend's Stream calls on the history shard of the Stream. A Workflow // Stream is on its Workflow's shard, and a Standalone Activity's Stream on its Activity's. service StreamService { rpc CreateStream(CreateStreamRequest) returns (CreateStreamResponse) { diff --git a/chasm/lib/stream/read.go b/chasm/lib/stream/read.go index 7513fd61d80..e6f229155a1 100644 --- a/chasm/lib/stream/read.go +++ b/chasm/lib/stream/read.go @@ -1,6 +1,7 @@ package stream import ( + "iter" "slices" "sort" "strings" @@ -36,14 +37,19 @@ func (s *Stream) StartOffset(req *streamservice.ReadRecordsRequest) (int64, erro from = s.NewestOffset + 1 } if from > s.NewestOffset+1 { - return 0, NewError(ReasonOffset, "The offset %d is past the newest offset + 1, %d", from, s.NewestOffset+1) + return 0, NewError( + ReasonOffset, + "The offset %d is past the newest offset + 1, %d", + from, + s.NewestOffset+1, + ) } return from, nil } -// Page reads the Records from the offset from. It is false when the page has no Record, so a -// reader waits. A page ends at the close Record, and a read past the close gives the close -// Record again, so every read of a closed Stream ends with it. +// Page reads the Records at or past from. Its bool is false when the page has no Record, so the +// reader waits. A read past the close gives the close Record again, so every read of a closed +// Stream ends with it. func (s *Stream) Page( ctx chasm.Context, req *streamservice.ReadRecordsRequest, @@ -76,7 +82,7 @@ func (s *Stream) Page( return resp, len(resp.GetRecords()) > 0, nil } -// scan collects the Records from from that pass the Topic filter, up to limit. +// scan collects the Records at or past from that pass the Topic filter, up to limit. func (s *Stream) scan( ctx chasm.Context, req *streamservice.ReadRecordsRequest, @@ -93,36 +99,56 @@ func (s *Stream) scan( resp := &streamservice.ReadRecordsResponse{NextOffset: from} scanned := 0 var size int64 - for _, info := range s.Index[s.firstBatchAt(start):] { - for _, entry := range s.Batches[info.GetFirstOffset()].Get(ctx).GetRecords() { - if entry.GetOffset() < start { - continue - } - if end := req.GetToOffset(); end > 0 && entry.GetOffset() >= end { - return resp, nil - } - if resp.Done || len(resp.Records) >= limit || scanned >= maxScan { - return resp, nil - } - if entry.GetOffset() >= retained && overBytes(req, resp, entry, size) { - return resp, nil + for entry := range s.entriesFrom(ctx, start) { + if end := req.GetToOffset(); end > 0 && entry.GetOffset() >= end { + return resp, nil + } + if pageFull(resp, limit, scanned) { + return resp, nil + } + if entry.GetOffset() >= retained && overBytes(req, resp, entry, size) { + return resp, nil + } + scanned++ + resp.NextOffset = entry.GetOffset() + 1 + if entry.GetOffset() < retained { + if from > 0 { + return nil, expired() } - scanned++ - resp.NextOffset = entry.GetOffset() + 1 - if entry.GetOffset() < retained { - if from > 0 { - return nil, expired() + continue + } + taken := len(resp.Records) + s.take(ctx, req, resp, entry) + if len(resp.Records) > taken { + size += int64(proto.Size(entry)) + } + } + return resp, nil +} + +// entriesFrom gives the stored entries at or past offset, oldest first. +func (s *Stream) entriesFrom( + ctx chasm.Context, + offset int64, +) iter.Seq[*streamservice.StreamRecord] { + return func(yield func(*streamservice.StreamRecord) bool) { + for _, info := range s.Index[s.firstBatchAt(offset):] { + for _, entry := range s.Batches[info.GetFirstOffset()].Get(ctx).GetRecords() { + if entry.GetOffset() >= offset && !yield(entry) { + return } - continue - } - taken := len(resp.Records) - s.take(ctx, req, resp, entry) - if len(resp.Records) > taken { - size += int64(proto.Size(entry)) } } } - return resp, nil +} + +// pageFull says whether a page ends before its next entry. An empty page doesn't stop at the scan +// cap, so a reader can't mistake it for the end while a match is left. +func pageFull(resp *streamservice.ReadRecordsResponse, limit int, scanned int) bool { + if len(resp.GetRecords()) == 0 { + return false + } + return resp.GetDone() || len(resp.GetRecords()) >= limit || scanned >= maxScan } // firstBatchAt is the index of the first batch with an entry at or past offset, so a look of a @@ -134,7 +160,7 @@ func (s *Stream) firstBatchAt(offset int64) int { } // overBytes says whether entry would take a page past its max_bytes. A page takes its first -// Record at any size, so a read always moves on. +// Record at any size, so a read always makes progress. func overBytes( req *streamservice.ReadRecordsRequest, resp *streamservice.ReadRecordsResponse, @@ -169,10 +195,9 @@ func (s *Stream) Last( var size int64 scanned := 0 collect := func() { - for i := len(s.Index) - 1; i >= 0; i-- { - entries := s.Batches[s.Index[i].GetFirstOffset()].Get(ctx).GetRecords() - for j := len(entries) - 1; j >= 0; j-- { - entry := entries[j] + for _, info := range slices.Backward(s.Index) { + entries := s.Batches[info.GetFirstOffset()].Get(ctx).GetRecords() + for _, entry := range slices.Backward(entries) { if entry.GetOffset() < retained || len(picked) >= want || scanned >= maxScan { return } @@ -248,10 +273,8 @@ func passes(req *streamservice.ReadRecordsRequest, topic string) bool { if len(req.GetTopics()) == 0 && len(req.GetTopicPrefixes()) == 0 { return true } - for _, t := range req.GetTopics() { - if t == topic { - return true - } + if slices.Contains(req.GetTopics(), topic) { + return true } for _, prefix := range req.GetTopicPrefixes() { if strings.HasPrefix(topic, prefix) { @@ -261,16 +284,21 @@ func passes(req *streamservice.ReadRecordsRequest, topic string) bool { return false } -// wakeAnswer is the answer of a wake_only read: no Records, the read's start as its next -// offset, and whether the Stream is closed. +// wakeAnswer is the answer of a wake_only read. It has no Records, and its next offset is the +// read's start. func (s *Stream) wakeAnswer(start int64) *streamservice.ReadRecordsResponse { - return &streamservice.ReadRecordsResponse{NextOffset: start, Done: s.Phase == streampb.STREAM_PHASE_CLOSED} + return &streamservice.ReadRecordsResponse{ + NextOffset: start, + Done: s.Phase == streampb.STREAM_PHASE_CLOSED, + } } -// afterClose gives the Superseded items written after the close from the offset from, and the -// offset after the last entry, zero if none. A Workflow Task that completes after the close -// still withdraws, and a reader stops at the close Record, so a page puts them before it. -func (s *Stream) afterClose(ctx chasm.Context, from, end int64) ([]*streamservice.StreamRecord, int64) { +// afterClose gives the Superseded items written after the close at or past from, and the offset +// after the last one. A reader stops at the close Record, so a page puts them before it. +func (s *Stream) afterClose( + ctx chasm.Context, + from, end int64, +) ([]*streamservice.StreamRecord, int64) { var records []*streamservice.StreamRecord var after int64 for _, info := range s.Index[s.firstBatchAt(from):] { @@ -317,6 +345,11 @@ type ReadInput struct { Limits Limits } +// parks says whether the read may park, a wake_only read whose operation a callback completes. +func (in ReadInput) parks() bool { + return in.Callback != nil && in.Request.GetWakeOnly() +} + // StartForRead is the start of an execution for a read. The Stream waits for its create. The // start parks the read itself, since a child node that the update of a new execution adds is // not persisted. @@ -328,19 +361,15 @@ func StartForRead(ctx chasm.MutableContext, in ReadInput) (*Stream, error) { return s, nil } -// ReadOrPark answers a read that has Records or ends, and parks any other read until the -// Stream can answer it. It answers nil for a parked read. -func (s *Stream) ReadOrPark(ctx chasm.MutableContext, in ReadInput) (*streamservice.ReadRecordsResponse, error) { +// ReadOrPark answers a read at once, but parks a wake_only read with no Record and answers nil. +// A wake's answer has no Records, so one append that wakes every parked read copies no page. +func (s *Stream) ReadOrPark( + ctx chasm.MutableContext, + in ReadInput, +) (*streamservice.ReadRecordsResponse, error) { s.applyDue(ctx, in.Limits) - if field, ok := s.Reads[in.RequestID]; ok { - read := field.Get(ctx) - if !read.GetResolved() { - return nil, nil - } - if read.GetFailure() != nil { - return nil, errorOfFailure(read.GetFailure()) - } - return common.CloneProto(read.GetResponse()), nil + if resp, known, err := s.parkedAnswer(ctx, in.RequestID); known { + return resp, err } now := ctx.Now(s) read := &ParkedRead{ @@ -353,41 +382,85 @@ func (s *Stream) ReadOrPark(ctx chasm.MutableContext, in ReadInput) (*streamserv if in.Reader != nil { read.ReaderRunId = in.Reader.RunID } + parks := in.parks() wait, createWait := parkWaits(in.Request, in.Limits) if wait > 0 { read.WaitDeadline = timestamppb.New(now.Add(wait)) } + s.reopen() if s.Phase == streampb.STREAM_PHASE_PENDING { - if createWait <= 0 || in.Callback == nil { + if createWait <= 0 || !parks { return nil, notFound() } read.CreateDeadline = timestamppb.New(now.Add(createWait)) - } else { - from, err := s.StartOffset(in.Request) - if err != nil { - return nil, err - } - if err := s.keepWakeFloor(ctx, in, from); err != nil { - return nil, err - } - resp, found, err := s.Page(ctx, in.Request, from) - if err != nil { - return nil, err - } - if found || in.Callback == nil { - s.finish(ctx, in.Limits) - if in.Request.GetWakeOnly() { - resp = s.wakeAnswer(from) - } - resp.ContinueAsNewSuggested = s.suggestsContinueAsNew(ctx, read.GetReaderRunId(), - in.Limits) - return resp, nil + } else if resp, err := s.firstLook(ctx, in, read); resp != nil || err != nil { + return resp, err + } + return nil, s.park(ctx, in, read, now) +} + +// parkedAnswer answers a retry of a read's start from the read that it parked. It is false when +// the Stream has no read of the request id. +func (s *Stream) parkedAnswer( + ctx chasm.Context, + requestID string, +) (*streamservice.ReadRecordsResponse, bool, error) { + field, ok := s.Reads[requestID] + if !ok { + return nil, false, nil + } + read := field.Get(ctx) + switch { + case !read.GetResolved(): + return nil, true, nil + case read.GetFailure() != nil: + return nil, true, errorOfFailure(read.GetFailure()) + default: + return common.CloneProto(read.GetResponse()), true, nil + } +} + +// firstLook pages a read of a Stream that exists. It answers the page when the read finds Records +// or doesn't park, and otherwise sets where the parked read looks next. +func (s *Stream) firstLook( + ctx chasm.MutableContext, + in ReadInput, + read *ParkedRead, +) (*streamservice.ReadRecordsResponse, error) { + from, err := s.StartOffset(in.Request) + if err != nil { + return nil, err + } + if err := s.keepWakeFloor(ctx, in, from); err != nil { + return nil, err + } + resp, found, err := s.Page(ctx, in.Request, from) + if err != nil { + return nil, err + } + if found || !in.parks() { + s.finish(ctx, in.Limits) + if in.Request.GetWakeOnly() { + resp = s.wakeAnswer(from) } - read.StartOffset = from - read.ScanOffset = max(from, resp.GetNextOffset()) + resp.ContinueAsNewSuggested = s.suggestsContinueAsNew(ctx, read.GetReaderRunId(), + in.Limits) + return resp, nil } + read.StartOffset = from + read.ScanOffset = max(from, resp.GetNextOffset()) + return nil, nil +} + +// park keeps the read with its completion callback until the Stream can answer it. +func (s *Stream) park( + ctx chasm.MutableContext, + in ReadInput, + read *ParkedRead, + now time.Time, +) error { if s.waitingReads(ctx) >= in.Limits.MaxParkedReads { - return nil, serviceerror.NewResourceExhausted( + return serviceerror.NewResourceExhausted( enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, "The Stream has the most waiting reads") } @@ -395,11 +468,13 @@ func (s *Stream) ReadOrPark(ctx chasm.MutableContext, in ReadInput) (*streamserv s.Reads = make(chasm.Map[string, *ParkedRead]) } s.Reads[in.RequestID] = chasm.NewComponentField(ctx, read) - if err := callback.Attach(ctx, read, in.RequestID, timestamppb.New(now), []*commonpb.Callback{in.Callback}); err != nil { - return nil, err + callbacks := []*commonpb.Callback{in.Callback} + err := callback.Attach(ctx, read, in.RequestID, timestamppb.New(now), callbacks) + if err != nil { + return err } s.finish(ctx, in.Limits) - return nil, nil + return nil } // CancelRead completes a parked read as canceled. @@ -419,8 +494,8 @@ type readOutcome struct { err error } -// wake answers a parked read when the Stream can. A look that finds only other Topics moves -// the read's offset on, so the next look starts after it. +// wake answers a parked read when the Stream can. A look that finds only other Topics advances +// the read's offset, so the next look starts after those entries. func (s *Stream) wake( ctx chasm.MutableContext, read *ParkedRead, @@ -435,7 +510,7 @@ func (s *Stream) wake( read.StartOffset = from read.ScanOffset = from } - // Nothing came since the read's last look, so every parked read doesn't rescan on each write. + // With no entry since the read's last look, a write doesn't rescan the parked read. if s.Phase == streampb.STREAM_PHASE_OPEN && read.GetScanOffset() > s.NewestOffset { return readOutcome{}, false } @@ -455,7 +530,10 @@ func (s *Stream) wake( } // resolveReads completes each waiting read for which decide is done. -func (s *Stream) resolveReads(ctx chasm.MutableContext, decide func(*ParkedRead) (readOutcome, bool)) { +func (s *Stream) resolveReads( + ctx chasm.MutableContext, + decide func(*ParkedRead) (readOutcome, bool), +) { for _, field := range s.Reads { read := field.Get(ctx) if read.GetResolved() { @@ -487,7 +565,8 @@ func (s *Stream) pruneReads(ctx chasm.MutableContext) { delivered := true for _, cb := range read.Callbacks { status := cb.Get(ctx).GetStatus() - if status != callbackspb.CALLBACK_STATUS_SUCCEEDED && status != callbackspb.CALLBACK_STATUS_FAILED { + if status != callbackspb.CALLBACK_STATUS_SUCCEEDED && + status != callbackspb.CALLBACK_STATUS_FAILED { delivered = false break } @@ -498,13 +577,13 @@ func (s *Stream) pruneReads(ctx chasm.MutableContext) { } } -// parkWaits are the waits of a parked read. A wake_only read may wait for days, since it adds -// nothing to History while it waits. Any other read waits as a long poll does. +// parkWaits are the waits of a parked read, which is a wake_only read. It may wait for days, +// since it adds nothing to History while it waits. func parkWaits( req *streamservice.ReadRecordsRequest, limits Limits, ) (wait time.Duration, createWait time.Duration) { - if !req.GetWakeOnly() || limits.MaxWakeWait <= 0 { + if limits.MaxWakeWait <= 0 { return ReadTimeout(req) } return min(max(req.GetWait().AsDuration(), 0), limits.MaxWakeWait), @@ -512,8 +591,12 @@ func parkWaits( } // ReadTimeout is how long a frontend read waits, bounded by the crate's limit. -func ReadTimeout(req *streamservice.ReadRecordsRequest) (wait time.Duration, createWait time.Duration) { - return min(max(req.GetWait().AsDuration(), 0), MaxWait), min(max(req.GetCreateWait().AsDuration(), 0), MaxWait) +func ReadTimeout( + req *streamservice.ReadRecordsRequest, +) (wait time.Duration, createWait time.Duration) { + wait = min(max(req.GetWait().AsDuration(), 0), MaxWait) + createWait = min(max(req.GetCreateWait().AsDuration(), 0), MaxWait) + return wait, createWait } // keepWakeFloor keeps the floor of a wake's Workflow in the transaction that takes the wake, diff --git a/chasm/lib/stream/readers.go b/chasm/lib/stream/readers.go index 74ae8583e25..c6f236689fd 100644 --- a/chasm/lib/stream/readers.go +++ b/chasm/lib/stream/readers.go @@ -25,7 +25,7 @@ type ReaderRun struct { } // readerRunOf finds the reading run in the links of a direct call. A call with no Workflow link, -// such as a replay's read again, keeps no floor. +// such as a read that a replay repeats, keeps no floor. func readerRunOf(links []nexus.Link) *ReaderRun { for _, link := range links { if event, err := commonnexus.ConvertNexusLinkToLinkWorkflowEvent(link); err == nil && @@ -36,12 +36,9 @@ func readerRunOf(links []nexus.Link) *ReaderRun { return nil } -// readerFloor is the floor that a read from from sets for the run. It is false when the Stream -// keeps no floors, when the run's floor is already that low, or when the read fails anyway. +// readerFloor is the floor that a read at from sets for the run. It's false when the run's floor +// is as low already, or when the read fails anyway. func (s *Stream) readerFloor(ctx chasm.Context, runID string, from int64) (int64, bool) { - if !s.Nested { - return 0, false - } if from == 0 { from = s.TrimmedOffset + 1 } @@ -54,9 +51,8 @@ func (s *Stream) readerFloor(ctx chasm.Context, runID string, from int64) (int64 return from, true } -// floorStart is where a run's floor starts for a read from start. A reset run copied the reads of -// the old run into its History, and its first live read gives the lowest of them in floor_from, -// so the floor covers what its replay reads again. +// floorStart is where a run's floor starts for a read at start. A reset run's first live read +// gives the lowest offset of its copied reads in floor_from, so the floor covers its replay. func (s *Stream) floorStart(req *streamservice.ReadRecordsRequest, start int64) int64 { floorFrom := req.GetFloorFrom() if floorFrom <= 0 || start == 0 || floorFrom >= start { @@ -82,9 +78,14 @@ func (s *Stream) suggestsContinueAsNew(ctx chasm.Context, runID string, limits L return 2*kept > limits.MaxStoredBytes } -// AddReader keeps the floor of a Workflow run that reads from from, in the transaction that +// AddReader keeps the floor of a Workflow run that reads at from, in the transaction that // answers the read. No trim passes the floor while the run is open. -func (s *Stream) AddReader(ctx chasm.MutableContext, run ReaderRun, from int64, limits Limits) error { +func (s *Stream) AddReader( + ctx chasm.MutableContext, + run ReaderRun, + from int64, + limits Limits, +) error { floor, ok := s.readerFloor(ctx, run.RunID, from) if !ok { return nil @@ -113,8 +114,8 @@ func (s *Stream) armReadersCheck(ctx chasm.MutableContext, limits Limits) { &streampb.StreamReadersCheckTask{Deadline: s.ReadersCheckTime}) } -// readersCheckTaskHandler drops the floors of the reader runs that closed. A run that continued -// as new is closed too, since the new run's History has none of its reads. +// readersCheckTaskHandler drops the floors and the waiting reads of the reader runs that closed. +// A run that continued as new is closed too, since the new run's History has none of its reads. type readersCheckTaskHandler struct { chasm.SideEffectTaskHandlerBase[*streampb.StreamReadersCheckTask] @@ -128,7 +129,11 @@ func newReadersCheckTaskHandler( namespaceRegistry namespace.Registry, historyClient resource.HistoryClient, ) *readersCheckTaskHandler { - return &readersCheckTaskHandler{config: config, namespaceRegistry: namespaceRegistry, historyClient: historyClient} + return &readersCheckTaskHandler{ + config: config, + namespaceRegistry: namespaceRegistry, + historyClient: historyClient, + } } func (h *readersCheckTaskHandler) Validate( @@ -137,7 +142,8 @@ func (h *readersCheckTaskHandler) Validate( _ chasm.TaskInvocation, task *streampb.StreamReadersCheckTask, ) (bool, error) { - return s.GetReadersCheckTime() != nil && s.GetReadersCheckTime().AsTime().Equal(task.GetDeadline().AsTime()), nil + return s.GetReadersCheckTime() != nil && + s.GetReadersCheckTime().AsTime().Equal(task.GetDeadline().AsTime()), nil } func (h *readersCheckTaskHandler) Execute( @@ -159,7 +165,12 @@ func (h *readersCheckTaskHandler) Execute( } var closed []string for runID, workflowID := range readers { - open, err := runOpen(ctx, h.historyClient, ref.NamespaceID, ReaderRun{WorkflowID: workflowID, RunID: runID}) + open, err := runOpen( + ctx, + h.historyClient, + ref.NamespaceID, + ReaderRun{WorkflowID: workflowID, RunID: runID}, + ) if err != nil { return err } @@ -183,9 +194,14 @@ func (h *readersCheckTaskHandler) Execute( return err } -// runOpen says whether a Workflow run is still open. A deleted or unknown run is closed. The run may be on -// another shard, so it goes through the history client. -func runOpen(ctx context.Context, historyClient resource.HistoryClient, namespaceID string, run ReaderRun) (bool, error) { +// runOpen says whether a Workflow run is still open. A deleted or unknown run is closed. The run +// may be on another shard, so the check calls the history client. +func runOpen( + ctx context.Context, + historyClient resource.HistoryClient, + namespaceID string, + run ReaderRun, +) (bool, error) { resp, err := historyClient.GetMutableState(ctx, &historyservice.GetMutableStateRequest{ NamespaceId: namespaceID, Execution: &commonpb.WorkflowExecution{WorkflowId: run.WorkflowID, RunId: run.RunID}, diff --git a/chasm/lib/stream/state.go b/chasm/lib/stream/state.go index e237ae3d3a2..0274496fa96 100644 --- a/chasm/lib/stream/state.go +++ b/chasm/lib/stream/state.go @@ -1,6 +1,7 @@ package stream import ( + "slices" "strings" "go.temporal.io/api/streamservice/v1" @@ -84,11 +85,17 @@ func (s *Stream) setReader(ctx chasm.MutableContext, run ReaderRun, offset int64 } } -// dropReaders drops the floors of runs that closed, and finds the lowest floor of the rest. +// dropReaders drops the floors and the waiting reads of runs that closed, and finds the lowest +// floor of the rest. A closed run takes no answer, and its wake would hold a slot for days. func (s *Stream) dropReaders(ctx chasm.MutableContext, runIDs []string) { for _, runID := range runIDs { delete(s.Readers, runID) } + if len(runIDs) > 0 { + s.resolveReads(ctx, func(read *ParkedRead) (readOutcome, bool) { + return readOutcome{err: errCanceled}, slices.Contains(runIDs, read.GetReaderRunId()) + }) + } s.Floor = 0 for runID := range s.Readers { floor := s.reader(ctx, runID) @@ -98,8 +105,8 @@ func (s *Stream) dropReaders(ctx chasm.MutableContext, runIDs []string) { } } -// limitProducers drops Producers past the cap. A Local Activity call writes only while it runs, -// so its entry goes first. Then the Producer with the oldest last batch goes, with its fence. +// limitProducers drops Producers past the cap, each with its fence. A Local Activity call writes +// only while it runs, so its entry is dropped first, then the one with the oldest last batch. func (s *Stream) limitProducers(ctx chasm.MutableContext, maxProducers int) { localPrefix := streamservice.PRODUCER_KIND_LOCAL_ACTIVITY.String() + "|" for maxProducers > 0 && len(s.Producers) > maxProducers { @@ -127,8 +134,8 @@ func (s *Stream) oldestProducer(ctx chasm.MutableContext, match func(string) boo return oldestKey } -// DropActivityProducers drops the entries of an Activity of a run that closed. It never writes -// again, so its fence and dedupe state can go. +// DropActivityProducers drops the entries of an Activity of a run that closed. The Activity never +// writes again, so its fence and dedupe state aren't needed. func (s *Stream) DropActivityProducers(runID, activityID string) { prefix := streamservice.PRODUCER_KIND_ACTIVITY.String() + "|act:" + encodePart(runID) + ":" + encodePart(activityID) + ":" diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index c074d8994a1..6e660344ef2 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -27,7 +27,7 @@ const ( MaxRecords = 1000 // MaxStreamIDBytes is the longest Stream id. MaxStreamIDBytes = 4096 - // The most entries one page scans, since a Topic filter can skip many. + // The most entries a page with Records scans, since a Topic filter can skip many. maxScan = 10 * MaxRecords // An offset is the append time in ms shifted by these bits, plus a sequence, as the Redis // Store packs its entry ids. So the offsets of the two backends sort the same way, and the @@ -146,6 +146,7 @@ func StartStream(ctx chasm.MutableContext, in CreateInput) (*Stream, error) { // Create opens the Stream, idempotently for one Retention. func (s *Stream) Create(ctx chasm.MutableContext, in CreateInput) (CreateOutput, error) { s.applyDue(ctx, in.Limits) + s.reopen() retention := in.Request.GetRetention().AsDuration() if retention <= 0 { retention = in.Limits.DefaultRetention @@ -168,8 +169,11 @@ func (s *Stream) Create(ctx chasm.MutableContext, in CreateInput) (CreateOutput, return CreateOutput{}, NewError(ReasonAlreadyExists, "The Stream exists with the Retention %v", s.GetRetention().AsDuration()) } - if owned(in.Request.GetOwner()) { + // The close check reads the owner, so the owner changes with WorkflowOwned, as the crate + // changes both. + if owned(in.Request.GetOwner()) && !s.WorkflowOwned { s.WorkflowOwned = true + s.Owner = in.Request.GetOwner() } s.withdrawOnCreate(ctx, in.Request) s.finish(ctx, in.Limits) @@ -183,8 +187,16 @@ func (s *Stream) Create(ctx chasm.MutableContext, in CreateInput) (CreateOutput, } } -// owned says whether a create's owner keeps the Stream open past one Retention with no write: a -// Workflow, or a Standalone Activity, which closes the Stream when it ends. +// reopen makes an abandoned nested Stream pending again for a create or a wait for the create. +// Only a Stream of its own has an execution that the caller can replace. +func (s *Stream) reopen() { + if s.Nested && s.Phase == streampb.STREAM_PHASE_ABANDONED { + s.Phase = streampb.STREAM_PHASE_PENDING + } +} + +// owned says whether a create's owner keeps the Stream open past one Retention with no write. A +// Workflow and a Standalone Activity do, since each closes its Stream when it ends. func owned(owner *streamservice.Producer) bool { kind := owner.GetKind() return kind == streamservice.PRODUCER_KIND_WORKFLOW || @@ -199,8 +211,11 @@ type AppendInput struct { // Append writes one batch of one Producer, with the checks and the withdrawal of the crate's // append script in one transition. The same epoch and sequence answers the first offsets -// again, so a retried batch goes in once. -func (s *Stream) Append(ctx chasm.MutableContext, in AppendInput) (*streamservice.AppendRecordsResponse, error) { +// again, so a retried batch is written once. +func (s *Stream) Append( + ctx chasm.MutableContext, + in AppendInput, +) (*streamservice.AppendRecordsResponse, error) { req := in.Request producer := req.GetProducer() if req.GetSequence() < 1 { @@ -215,31 +230,17 @@ func (s *Stream) Append(ctx chasm.MutableContext, in AppendInput) (*streamservic } key := producerKey(producer) digest := BatchDigest(req.GetRecords()) - known := s.producer(ctx, key) - if known != nil && CompareEpochs(producer.GetEpoch(), known.GetEpoch()) == 0 { - if req.GetSequence() == known.GetSequence() { - if len(known.GetDigest()) > 0 && !bytes.Equal(known.GetDigest(), digest) { - return nil, NewError(ReasonSeqDivergent, - "The sequence %d has other content than its batch", req.GetSequence()) - } - return &streamservice.AppendRecordsResponse{FirstOffset: known.GetFirstOffset(), LastOffset: known.GetLastOffset()}, nil - } - if req.GetSequence() < known.GetSequence() { - return nil, NewError(ReasonSequence, - "The sequence %d is below the last one, %d", req.GetSequence(), known.GetSequence()) - } - } - // A Workflow batch with no Records only settles its task facts, which a closed Stream - // still takes, as the Superseded Records after the close. - deferred := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && req.GetDeferWithdrawal() - settles := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && len(req.GetRecords()) == 0 && !deferred + if resp, err := resend(req, s.producer(ctx, key), digest); resp != nil || err != nil { + return resp, err + } + w := withdrawalOf(req) + // A Workflow batch with no Records only settles its task facts. A closed Stream still takes + // it, since a Workflow Task that completes after the close can still withdraw Records. + settles := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && + len(req.GetRecords()) == 0 && !w.Defer if s.Phase == streampb.STREAM_PHASE_CLOSED && !settles { return nil, NewError(ReasonClosed, "The Stream is closed") } - w := Withdrawal{WithdrawRun: req.GetWithdrawRun(), Defer: deferred} - if producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW { - w.PreviousCompleted, w.FailedTasks = req.GetPreviousCompletedTask(), req.GetFailedTasks() - } flagged, err := s.admit(ctx, producer, w) if err != nil { return nil, err @@ -264,22 +265,14 @@ func (s *Stream) Append(ctx chasm.MutableContext, in AppendInput) (*streamservic if orphans { s.settleOrphans(ctx, producer, orphaned, wrote) } - state := &streampb.ProducerState{ + s.setProducer(ctx, key, &streampb.ProducerState{ Epoch: producer.GetEpoch(), Sequence: req.GetSequence(), FirstOffset: first, LastOffset: last, - Holds: s.producer(ctx, key).GetHolds(), + Holds: newHolds(producer, s.producer(ctx, key).GetHolds(), wrote, flagged), Digest: digest, - } - if withdrawsEpochs(producer.GetKind()) { - if wrote { - state.Holds = producer.GetEpoch() - } else if flagged { - state.Holds = nil - } - } - s.setProducer(ctx, key, state) + }) if wrote { s.LastWriteTime = timestamppb.New(ctx.Now(s)) } @@ -288,6 +281,62 @@ func (s *Stream) Append(ctx chasm.MutableContext, in AppendInput) (*streamservic return &streamservice.AppendRecordsResponse{FirstOffset: first, LastOffset: last}, nil } +// resend answers a resend of a Producer's last batch with that batch's offsets, and refuses a +// lower sequence or other content. It answers nil for a new batch. +func resend( + req *streamservice.AppendRecordsRequest, + known *streampb.ProducerState, + digest []byte, +) (*streamservice.AppendRecordsResponse, error) { + if known == nil || CompareEpochs(req.GetProducer().GetEpoch(), known.GetEpoch()) != 0 { + return nil, nil + } + switch { + case req.GetSequence() > known.GetSequence(): + return nil, nil + case req.GetSequence() < known.GetSequence(): + return nil, NewError(ReasonSequence, + "The sequence %d is below the last one, %d", req.GetSequence(), known.GetSequence()) + case len(known.GetDigest()) > 0 && !bytes.Equal(known.GetDigest(), digest): + return nil, NewError(ReasonSeqDivergent, + "The sequence %d has other content than its batch", req.GetSequence()) + default: + return &streamservice.AppendRecordsResponse{ + FirstOffset: known.GetFirstOffset(), + LastOffset: known.GetLastOffset(), + }, nil + } +} + +// withdrawalOf is what a batch withdraws. Only a Workflow batch knows its Workflow Task facts. +func withdrawalOf(req *streamservice.AppendRecordsRequest) Withdrawal { + w := Withdrawal{WithdrawRun: req.GetWithdrawRun()} + if req.GetProducer().GetKind() == streamservice.PRODUCER_KIND_WORKFLOW { + w.Defer = req.GetDeferWithdrawal() + w.PreviousCompleted, w.FailedTasks = req.GetPreviousCompletedTask(), req.GetFailedTasks() + } + return w +} + +// newHolds is the epoch whose Records the Producer's next higher epoch withdraws. A batch with +// Records holds its own epoch, and a batch that withdrew the held Records holds none. +func newHolds( + producer *streamservice.Producer, + held *streamservice.ProducerEpoch, + wrote, flagged bool, +) *streamservice.ProducerEpoch { + switch { + case !withdrawsEpochs(producer.GetKind()): + return held + case wrote: + return producer.GetEpoch() + case flagged: + return nil + default: + return held + } +} + // BatchDigest is the SHA-256 of a batch's content, as the crate and the memory Store compute it. // Each Record gives its Topic, its metadata count, its metadata sorted by key, and its data, each // part with a big-endian u64 length first. A resend with the same sequence must have the same @@ -317,13 +366,17 @@ func BatchDigest(records []*streamservice.AppendRecord) []byte { return h.Sum(nil) } -// batchEntries are the entries of one write: the orphans' Superseded Record, the batch's own -// Superseded Record, then the Records. -func batchEntries(req *streamservice.AppendRecordsRequest, w Withdrawal, flagged, orphans bool) []*streamservice.StreamRecord { +// batchEntries are the entries of one write. The orphans' Superseded Record comes first, then the +// batch's own Superseded Record, then the Records. +func batchEntries( + req *streamservice.AppendRecordsRequest, + w Withdrawal, + flagged, orphans bool, +) []*streamservice.StreamRecord { producer := req.GetProducer() var entries []*streamservice.StreamRecord - // The orphans go in an entry of their own, since the batch's first entry may carry its own - // Superseded Record. + // The orphans get an entry of their own, since the batch may also write its own Superseded + // Record. if orphans { entries = append(entries, orphanRecord(req)) } @@ -345,16 +398,30 @@ func batchEntries(req *streamservice.AppendRecordsRequest, w Withdrawal, flagged // readers keep their Records, or one that its Workflow can't take. func (s *Stream) checkSize(ctx chasm.MutableContext, size int64, limits Limits) error { if size > limits.MaxBatchBytes { - return NewError(ReasonTooLarge, "The batch has %d bytes, more than %d", size, limits.MaxBatchBytes) + return NewError( + ReasonTooLarge, + "The batch has %d bytes, more than %d", + size, + limits.MaxBatchBytes, + ) } if kept := s.keptBytes(); kept+size > limits.MaxStoredBytes { - return NewError(ReasonTooLarge, "The Stream keeps %d bytes for its readers, and the cap is %d", kept, limits.MaxStoredBytes) + return NewError( + ReasonTooLarge, + "The Stream keeps %d bytes for its readers, and the cap is %d", + kept, + limits.MaxStoredBytes, + ) } // The Records of a Workflow Stream are in the Workflow's mutable state, which terminates the // run past its own limit, so the write fails first. if limits.MaxWorkflowStateBytes > 0 && int64(ctx.ExecutionInfo().ApproximateStateSize)+size > limits.MaxWorkflowStateBytes { - return NewError(ReasonTooLarge, "The Workflow state would pass %d bytes", limits.MaxWorkflowStateBytes) + return NewError( + ReasonTooLarge, + "The Workflow state would pass %d bytes", + limits.MaxWorkflowStateBytes, + ) } return nil } @@ -363,11 +430,18 @@ func (s *Stream) checkSize(ctx chasm.MutableContext, size int64, limits Limits) type CloseInput struct { Request *streamservice.CloseStreamRequest Limits Limits + // Command is a Workflow run's System Nexus close, which the server runs once the Workflow + // Task that issued it commits. Its Producer has no epoch, so it fences nothing and withdraws + // nothing. + Command bool } // Close writes the close Record, with the Producer checks and the withdrawal of a batch. A // second close does nothing. -func (s *Stream) Close(ctx chasm.MutableContext, in CloseInput) (*streamservice.CloseStreamResponse, error) { +func (s *Stream) Close( + ctx chasm.MutableContext, + in CloseInput, +) (*streamservice.CloseStreamResponse, error) { req := in.Request producer := req.GetProducer() status, err := callerStatus(req) @@ -375,24 +449,38 @@ func (s *Stream) Close(ctx chasm.MutableContext, in CloseInput) (*streamservice. return nil, err } s.applyDue(ctx, in.Limits) + // The owner check goes first, as in the crate, so a second close succeeds only for a caller + // that may close. + if s.Exists() { + if err := s.checkCloser(producer); err != nil { + return nil, err + } + } if s.Phase == streampb.STREAM_PHASE_CLOSED { return &streamservice.CloseStreamResponse{}, nil } if err := s.writable(); err != nil { return nil, err } - if err := s.checkCloser(producer); err != nil { - return nil, err - } - w := Withdrawal{PreviousCompleted: req.GetPreviousCompletedTask(), FailedTasks: req.GetFailedTasks(), WithdrawRun: req.GetWithdrawRun()} - flagged, err := s.admit(ctx, producer, w) - if err != nil { - return nil, err - } - if flagged { - s.write(ctx, []*streamservice.StreamRecord{supersededRecord(producer, w)}) + if in.Command { + if s.runFenced(producer.GetRunId()) { + return nil, NewError(ReasonFenced, "A Workflow retry withdrew the run of the Producer") + } + } else { + w := Withdrawal{ + PreviousCompleted: req.GetPreviousCompletedTask(), + FailedTasks: req.GetFailedTasks(), + WithdrawRun: req.GetWithdrawRun(), + } + flagged, err := s.admit(ctx, producer, w) + if err != nil { + return nil, err + } + if flagged { + s.write(ctx, []*streamservice.StreamRecord{supersededRecord(producer, w)}) + } + s.settle(ctx, producer, w, false) } - s.settle(ctx, producer, w, false) s.close(ctx, producer, req.GetResult(), req.GetFailure(), status) s.finish(ctx, in.Limits) return &streamservice.CloseStreamResponse{}, nil @@ -452,17 +540,28 @@ func (s *Stream) writable() error { } } -// withdrawOnCreate applies a retry run's withdraw_run at its first create. -func (s *Stream) withdrawOnCreate(ctx chasm.MutableContext, req *streamservice.CreateStreamRequest) { - w := Withdrawal{WithdrawRun: req.GetWithdrawRun()} - if s.withdrawsRun(ctx, w.WithdrawRun) { +// withdrawOnCreate applies a retry run's withdraw_run at its first create. A create has no task +// facts, so it withdraws no task, and its Superseded Record has no epoch, as the crate's. +func (s *Stream) withdrawOnCreate( + ctx chasm.MutableContext, + req *streamservice.CreateStreamRequest, +) { + run := req.GetWithdrawRun() + if run == "" || s.runFenced(run) { + return + } + if s.withdrawsRun(ctx, run) { + w := Withdrawal{WithdrawRun: run, Defer: true} s.write(ctx, []*streamservice.StreamRecord{supersededRecord(req.GetOwner(), w)}) } - s.settle(ctx, req.GetOwner(), w, false) + s.FencedRuns = append(s.FencedRuns, run) } // write gives the entries their offsets and stores them as one batch. -func (s *Stream) write(ctx chasm.MutableContext, entries []*streamservice.StreamRecord) (first int64, last int64) { +func (s *Stream) write( + ctx chasm.MutableContext, + entries []*streamservice.StreamRecord, +) (first int64, last int64) { if len(entries) == 0 { return 0, 0 } @@ -478,7 +577,10 @@ func (s *Stream) write(ctx chasm.MutableContext, entries []*streamservice.Stream } s.Batches[first] = chasm.NewDataField(ctx, batch) size := int64(proto.Size(batch)) - s.Index = append(s.Index, &streampb.BatchInfo{FirstOffset: first, LastOffset: last, Bytes: size}) + s.Index = append( + s.Index, + &streampb.BatchInfo{FirstOffset: first, LastOffset: last, Bytes: size}, + ) s.StoredBytes += size s.NewestOffset = last return first, last @@ -578,11 +680,12 @@ func (s *Stream) floor() (int64, bool) { return s.Floor, s.Floor > 0 && len(s.Readers) > 0 } -// retainedFrom is the offset below which the Retention hides the Records at now. A Workflow Stream -// hides none that it still stores, so a read from the start begins at the same offset on replay. +// retainedFrom is the offset below which the Retention hides the Records at now. A nested Stream +// or one with floors hides none it stores, so a replay of a read from the start starts the same. func (s *Stream) retainedFrom(now time.Time) int64 { retained := s.TrimmedOffset + 1 - if retention := s.GetRetention().AsDuration(); retention > 0 && !s.Nested { + retention := s.GetRetention().AsDuration() + if retention > 0 && !s.Nested && len(s.Readers) == 0 { if cutMs := now.Add(-retention).UnixMilli(); cutMs > 0 { retained = max(retained, cutMs< 0 && len(s.Index) > 0 && !s.kept(s.Index[0]) { - consider(time.UnixMilli(s.Index[0].GetLastOffset() >> seqBits).Add(retention + time.Millisecond)) + if retention := s.GetRetention().AsDuration(); retention > 0 && len(s.Index) > 0 && + !s.kept(s.Index[0]) { + consider( + time.UnixMilli(s.Index[0].GetLastOffset() >> seqBits).Add(retention + time.Millisecond), + ) } return deadline, found } -// finish wakes the reads that can answer now, drops the reads whose answer went out, applies +// finish wakes the reads that can answer now, drops the reads that have answered, applies // the size cap and arms the timer for the next deadline. func (s *Stream) finish(ctx chasm.MutableContext, limits Limits) { s.limitStored(limits.MaxStoredBytes) @@ -698,7 +807,7 @@ func (s *Stream) finish(ctx chasm.MutableContext, limits Limits) { return s.wake(ctx, read, limits) }) } else if s.waitingReads(ctx) == 0 { - // No read waits for the create any more, so the execution can go. + // No read waits for the create any more, so the Stream is done. s.Phase = streampb.STREAM_PHASE_ABANDONED } s.pruneReads(ctx) @@ -719,7 +828,10 @@ func (s *Stream) finish(ctx chasm.MutableContext, limits Limits) { // producerKey keys the Producer table by kind, id and run. func producerKey(producer *streamservice.Producer) string { - return strings.Join([]string{producer.GetKind().String(), producer.GetId(), producer.GetRunId()}, "|") + return strings.Join( + []string{producer.GetKind().String(), producer.GetId(), producer.GetRunId()}, + "|", + ) } // CompareEpochs compares two epochs field by field, in the order of the crate. The order is the diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index 2c68f1334a8..2c196ebffe9 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "encoding/json" + "fmt" "os" "testing" "time" @@ -21,6 +22,7 @@ import ( "go.temporal.io/server/common/clock" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/testing/protorequire" "google.golang.org/grpc/codes" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" @@ -40,7 +42,10 @@ func newHarness(t *testing.T) *harness { require.NoError(t, registry.Register(&chasm.CoreLibrary{})) require.NoError(t, registry.Register(callback.NewNilLibrary())) config := ConfigProvider(dynamicconfig.NewNoopCollection()) - require.NoError(t, registry.Register(newLibrary(config, nil, nil, newTimerTaskHandler(config), nil, nil))) + require.NoError( + t, + registry.Register(newLibrary(config, nil, nil, newTimerTaskHandler(config), nil, nil)), + ) ts := clock.NewEventTimeSource() ts.Update(time.Date(2026, 10, 22, 0, 0, 0, 0, time.UTC)) engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(ts)) @@ -59,16 +64,27 @@ func (h *harness) ref() chasm.ComponentRef { } func (h *harness) create(req *streamservice.CreateStreamRequest) (CreateOutput, error) { - result, err := chasm.UpdateWithStartExecution(h.ctx, h.key, StartStream, (*Stream).Create, + result, err := chasm.UpdateWithStartExecution( + h.ctx, + h.key, + StartStream, + (*Stream).Create, CreateInput{Request: req, Limits: h.limits}, - chasm.WithBusinessIDPolicy(chasm.BusinessIDReusePolicyRejectDuplicate, chasm.BusinessIDConflictPolicyUseExisting), + chasm.WithBusinessIDPolicy( + chasm.BusinessIDReusePolicyRejectDuplicate, + chasm.BusinessIDConflictPolicyUseExisting, + ), ) out := result.UpdateOutput out.Created = out.Created || result.Created return out, err } -func (h *harness) append(producer *streamservice.Producer, sequence int64, topics ...string) (*streamservice.AppendRecordsResponse, error) { +func (h *harness) append( + producer *streamservice.Producer, + sequence int64, + topics ...string, +) (*streamservice.AppendRecordsResponse, error) { req := &streamservice.AppendRecordsRequest{Producer: producer, Sequence: sequence} for _, topic := range topics { req.Records = append(req.Records, &streamservice.AppendRecord{ @@ -79,8 +95,15 @@ func (h *harness) append(producer *streamservice.Producer, sequence int64, topic return h.appendRequest(req) } -func (h *harness) appendRequest(req *streamservice.AppendRecordsRequest) (*streamservice.AppendRecordsResponse, error) { - resp, _, err := chasm.UpdateComponent(h.ctx, h.ref(), (*Stream).Append, AppendInput{Request: req, Limits: h.limits}) +func (h *harness) appendRequest( + req *streamservice.AppendRecordsRequest, +) (*streamservice.AppendRecordsResponse, error) { + resp, _, err := chasm.UpdateComponent( + h.ctx, + h.ref(), + (*Stream).Append, + AppendInput{Request: req, Limits: h.limits}, + ) return resp, err } @@ -95,31 +118,68 @@ func (h *harness) close(result string) error { return err } -func (h *harness) read(req *streamservice.ReadRecordsRequest) (*streamservice.ReadRecordsResponse, error) { - return chasm.ReadComponent(h.ctx, h.ref(), - func(s *Stream, ctx chasm.Context, req *streamservice.ReadRecordsRequest) (*streamservice.ReadRecordsResponse, error) { +// commandClose closes the Stream with the System Nexus command of run. +func (h *harness) commandClose(run, result string) error { + _, _, err := chasm.UpdateComponent(h.ctx, h.ref(), (*Stream).Close, CloseInput{ + Request: &streamservice.CloseStreamRequest{ + Producer: &streamservice.Producer{ + Kind: streamservice.PRODUCER_KIND_WORKFLOW, + Id: "wf:" + run, + RunId: run, + }, + Result: &commonpb.Payload{Data: []byte(result)}, + }, + Limits: h.limits, + Command: true, + }) + return err +} + +func (h *harness) read( + req *streamservice.ReadRecordsRequest, +) (*streamservice.ReadRecordsResponse, error) { + return chasm.ReadComponent( + h.ctx, + h.ref(), + func( + s *Stream, + ctx chasm.Context, + req *streamservice.ReadRecordsRequest, + ) (*streamservice.ReadRecordsResponse, error) { from, err := s.StartOffset(req) if err != nil { return nil, err } resp, _, err := s.Page(ctx, req, from) return resp, err - }, req) + }, + req, + ) } -func (h *harness) readOrPark(requestID string, req *streamservice.ReadRecordsRequest) (*streamservice.ReadRecordsResponse, error) { - result, err := chasm.UpdateWithStartExecution(h.ctx, h.key, +func (h *harness) readOrPark( + requestID string, + req *streamservice.ReadRecordsRequest, +) (*streamservice.ReadRecordsResponse, error) { + result, err := chasm.UpdateWithStartExecution( + h.ctx, + h.key, StartForRead, (*Stream).ReadOrPark, ReadInput{ Request: req, RequestID: requestID, - Callback: &commonpb.Callback{Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{ - Url: chasm.NexusCompletionHandlerURL, - }}}, + Callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{ + Url: chasm.NexusCompletionHandlerURL, + }}, + }, Limits: h.limits, }, - chasm.WithBusinessIDPolicy(chasm.BusinessIDReusePolicyRejectDuplicate, chasm.BusinessIDConflictPolicyUseExisting), + chasm.WithBusinessIDPolicy( + chasm.BusinessIDReusePolicyRejectDuplicate, + chasm.BusinessIDConflictPolicyUseExisting, + ), ) return result.UpdateOutput, err } @@ -195,7 +255,12 @@ func TestCreateIsIdempotentForOneRetention(t *testing.T) { require.NoError(t, err) require.Equal(t, CreateOutput{Status: streamservice.STREAM_STATUS_OPEN}, out) - _, err = h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Retention: durationpb.New(time.Hour)}) + _, err = h.create( + &streamservice.CreateStreamRequest{ + StreamId: "chat-1", + Retention: durationpb.New(time.Hour), + }, + ) require.ErrorContains(t, err, ReasonAlreadyExists) } @@ -214,11 +279,18 @@ func TestAppendAndRead(t *testing.T) { require.Equal(t, []string{"turns", "tool/search", "other"}, topics(all)) require.Equal(t, first.GetLastOffset()+1, all.GetNextOffset()) - filtered, err := h.read(&streamservice.ReadRecordsRequest{Topics: []string{"turns"}, TopicPrefixes: []string{"tool/"}}) + filtered, err := h.read( + &streamservice.ReadRecordsRequest{ + Topics: []string{"turns"}, + TopicPrefixes: []string{"tool/"}, + }, + ) require.NoError(t, err) require.Equal(t, []string{"turns", "tool/search"}, topics(filtered)) - page, err := h.read(&streamservice.ReadRecordsRequest{FromOffset: first.GetFirstOffset() + 1, MaxRecords: 1}) + page, err := h.read( + &streamservice.ReadRecordsRequest{FromOffset: first.GetFirstOffset() + 1, MaxRecords: 1}, + ) require.NoError(t, err) require.Equal(t, []string{"tool/search"}, topics(page)) require.Equal(t, first.GetFirstOffset()+2, page.GetNextOffset()) @@ -286,9 +358,167 @@ func TestCloseEndsEveryRead(t *testing.T) { require.Equal(t, CreateOutput{Status: streamservice.STREAM_STATUS_CLOSED}, out) } +// A Workflow owns a Stream only in its chain, where the chain checks and the chain's end apply. +func TestWorkflowOwnerNeedsAWorkflowRef(t *testing.T) { + require.ErrorContains(t, checkOwner(workflowProducer(1), target{}), ReasonInvalid) + nested := target{nested: true, host: chasm.WorkflowArchetypeID} + require.NoError(t, checkOwner(workflowProducer(1), nested)) +} + +// A Workflow's create of an open client Stream makes the Workflow its owner, so a client can't +// close it any more. +func TestWorkflowCreateTakesOverTheOwner(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + _, err = h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(1), + }) + require.NoError(t, err) + + require.ErrorContains(t, h.close("done"), ReasonNotOwner) + require.NoError(t, h.commandClose("run", "done")) +} + +// A create has no task facts. A create again in a later task of the same run withdraws none of +// the run's Records, so a last read gives them as a plain read does. +func TestCreateAgainKeepsTheRunsRecordsInLast(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(3), + }) + require.NoError(t, err) + for i, task := range []int64{3, 7} { + call := localActivityProducer(task) + call.Id = fmt.Sprintf("la:run:%d", task) + _, err := h.append(call, int64(i+1), "t") + require.NoError(t, err) + } + _, err = h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(12), + }) + require.NoError(t, err) + + plain, err := h.read(&streamservice.ReadRecordsRequest{}) + require.NoError(t, err) + require.Equal(t, []string{"t", "t"}, topics(plain)) + last, err := chasm.ReadComponent(h.ctx, h.ref(), (*Stream).Last, + &streamservice.ReadRecordsRequest{Last: 10}) + require.NoError(t, err) + require.Equal(t, []string{"t", "t"}, topics(last)) +} + +// A retry run's create withdraws its previous run with a Superseded Record that has no epoch, as +// the crate's, so a reader withdraws no task of the retry run. +func TestCreateWithdrawsThePreviousRunWithNoEpoch(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(1), + }) + require.NoError(t, err) + _, err = h.append(workflowProducer(3), 1, "a") + require.NoError(t, err) + retry := workflowProducer(5) + retry.Id, retry.RunId = "wf:retry", "retry" + _, err = h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: retry, WithdrawRun: "run", + }) + require.NoError(t, err) + + read, err := h.read(&streamservice.ReadRecordsRequest{}) + require.NoError(t, err) + require.Equal(t, []string{"a", "superseded"}, topics(read)) + supersession := read.GetRecords()[1].GetSupersession() + require.Nil(t, supersession.GetEpoch()) + require.Equal(t, "run", supersession.GetWithdrawRun()) + _, err = h.append(workflowProducer(9), 2, "late") + require.ErrorContains(t, err, ReasonFenced) +} + +// A second close does nothing, but only for a caller that may close the Stream. +func TestSecondCloseChecksTheOwner(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + require.NoError(t, h.close("done")) + _, _, err = chasm.UpdateComponent(h.ctx, h.ref(), (*Stream).Close, CloseInput{ + Request: &streamservice.CloseStreamRequest{Producer: activityProducer(1)}, + Limits: h.limits, + }) + require.ErrorContains(t, err, ReasonNotOwner) + require.NoError(t, h.close("again")) +} + +// A command close has no epoch, so the epoch of the run's earlier writes doesn't fence it. +func TestCommandCloseOfTheOwnerRun(t *testing.T) { + h := newHarness(t) + _, err := h.create( + &streamservice.CreateStreamRequest{StreamId: "chat-1", Owner: workflowProducer(1)}, + ) + require.NoError(t, err) + _, err = h.append(workflowProducer(3), 1, "a") + require.NoError(t, err) + require.NoError(t, h.commandClose("run", "done")) + require.NoError(t, h.commandClose("run", "again")) + + all, err := h.read(&streamservice.ReadRecordsRequest{}) + require.NoError(t, err) + require.Equal(t, []string{"a", "close"}, topics(all)) + closed := all.GetRecords()[1] + require.Equal(t, "done", string(closed.GetCloseResult().GetData())) + protorequire.ProtoEqual(t, &streamservice.Producer{ + Kind: streamservice.PRODUCER_KIND_WORKFLOW, + Id: "wf:run", + RunId: "run", + }, closed.GetProducer()) +} + +// A Local Activity writes before its Workflow Task commits, so it can't close a Workflow Stream. +// A failed task would leave the Stream closed, and the retried task's create would fail. +func TestLocalActivityCantCloseAWorkflowStream(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(1), + }) + require.NoError(t, err) + _, _, err = chasm.UpdateComponent(h.ctx, h.ref(), (*Stream).Close, CloseInput{ + Request: &streamservice.CloseStreamRequest{Producer: localActivityProducer(3)}, + Limits: h.limits, + }) + require.ErrorContains(t, err, ReasonNotOwner) + + require.NoError(t, h.commandClose("run", "done")) +} + +func TestCommandCloseOfAWithdrawnRunIsFenced(t *testing.T) { + h := newHarness(t) + _, err := h.create( + &streamservice.CreateStreamRequest{StreamId: "chat-1", Owner: workflowProducer(1)}, + ) + require.NoError(t, err) + retry := workflowProducer(3) + retry.Id, retry.RunId = "wf:retry", "retry" + _, err = h.appendRequest( + &streamservice.AppendRecordsRequest{ + Producer: retry, + Sequence: 1, + Records: records("a"), + WithdrawRun: "run", + }, + ) + require.NoError(t, err) + + require.ErrorContains(t, h.commandClose("run", "done"), ReasonFenced) +} + func TestRetentionDropsOldRecordsAndExpiresTheStream(t *testing.T) { h := newHarness(t) - _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Retention: durationpb.New(time.Hour)}) + _, err := h.create( + &streamservice.CreateStreamRequest{ + StreamId: "chat-1", + Retention: durationpb.New(time.Hour), + }, + ) require.NoError(t, err) old, err := h.append(activityProducer(1), 1, "old") require.NoError(t, err) @@ -306,17 +536,24 @@ func TestRetentionDropsOldRecordsAndExpiresTheStream(t *testing.T) { h.advance(time.Hour) _, err = h.read(&streamservice.ReadRecordsRequest{}) require.ErrorContains(t, err, ReasonExpired) - out, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Retention: durationpb.New(time.Hour)}) + out, err := h.create( + &streamservice.CreateStreamRequest{ + StreamId: "chat-1", + Retention: durationpb.New(time.Hour), + }, + ) require.Error(t, err) require.Equal(t, CreateOutput{}, out) } +// A wake answers with no Records once one passes its filter, and the reader reads the page. func TestParkedReadWakesOnAppend(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) require.NoError(t, err) - resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{Topics: []string{"turns"}}) + wake := &streamservice.ReadRecordsRequest{Topics: []string{"turns"}, WakeOnly: true} + resp, err := h.readOrPark("read-1", wake) require.NoError(t, err) require.Nil(t, resp) require.False(t, h.parked("read-1").GetResolved()) @@ -329,19 +566,24 @@ func TestParkedReadWakesOnAppend(t *testing.T) { require.NoError(t, err) read := h.parked("read-1") require.True(t, read.GetResolved()) - require.Equal(t, []string{"turns"}, topics(read.GetResponse())) - require.Equal(t, []callbackspb.CallbackStatus{callbackspb.CALLBACK_STATUS_SCHEDULED}, read.callbacks) + require.Empty(t, read.GetResponse().GetRecords()) + require.Equal( + t, + []callbackspb.CallbackStatus{callbackspb.CALLBACK_STATUS_SCHEDULED}, + read.callbacks, + ) - again, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{Topics: []string{"turns"}}) + again, err := h.readOrPark("read-1", wake) require.NoError(t, err) - require.Equal(t, []string{"turns"}, topics(again)) + require.Empty(t, again.GetRecords()) + require.Equal(t, int64(0), again.GetNextOffset()) } func TestParkedReadWakesOnClose(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) require.NoError(t, err) - resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{}) + resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{WakeOnly: true}) require.NoError(t, err) require.Nil(t, resp) @@ -366,10 +608,12 @@ func TestReadWithRecordsAnswersAtOnce(t *testing.T) { func TestReadWaitsForTheCreate(t *testing.T) { h := newHarness(t) - _, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{}) + _, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{WakeOnly: true}) require.ErrorContains(t, err, ReasonNotFound) - resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{CreateWait: durationpb.New(30 * time.Second)}) + resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{ + WakeOnly: true, CreateWait: durationpb.New(30 * time.Second), + }) require.NoError(t, err) require.Nil(t, resp) _, err = h.append(activityProducer(1), 1, "a") @@ -382,12 +626,14 @@ func TestReadWaitsForTheCreate(t *testing.T) { _, err = h.append(activityProducer(1), 1, "a") require.NoError(t, err) - require.Equal(t, []string{"a"}, topics(h.parked("read-1").GetResponse())) + require.True(t, h.parked("read-1").GetResolved()) } func TestReadForTheCreateTimesOut(t *testing.T) { h := newHarness(t) - resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{CreateWait: durationpb.New(30 * time.Second)}) + resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{ + WakeOnly: true, CreateWait: durationpb.New(30 * time.Second), + }) require.NoError(t, err) require.Nil(t, resp) @@ -409,9 +655,13 @@ func workflowProducer(started int64) *streamservice.Producer { func localActivityProducer(started int64) *streamservice.Producer { return &streamservice.Producer{ - Kind: streamservice.PRODUCER_KIND_LOCAL_ACTIVITY, - Id: "la:run:1", - Epoch: &streamservice.ProducerEpoch{TaskStartedEventId: started, TaskAttempt: 1, Attempt: 1}, + Kind: streamservice.PRODUCER_KIND_LOCAL_ACTIVITY, + Id: "la:run:1", + Epoch: &streamservice.ProducerEpoch{ + TaskStartedEventId: started, + TaskAttempt: 1, + Attempt: 1, + }, RunId: "run", } } @@ -419,7 +669,10 @@ func localActivityProducer(started int64) *streamservice.Producer { func records(topics ...string) []*streamservice.AppendRecord { var out []*streamservice.AppendRecord for _, topic := range topics { - out = append(out, &streamservice.AppendRecord{Topic: topic, Body: &commonpb.Payload{Data: []byte(topic)}}) + out = append( + out, + &streamservice.AppendRecord{Topic: topic, Body: &commonpb.Payload{Data: []byte(topic)}}, + ) } return out } @@ -462,7 +715,14 @@ func TestWithdrawRunFencesThePreviousRun(t *testing.T) { retry := workflowProducer(3) retry.Id, retry.RunId = "wf:retry", "retry" - _, err = h.appendRequest(&streamservice.AppendRecordsRequest{Producer: retry, Sequence: 1, Records: records("b"), WithdrawRun: "run"}) + _, err = h.appendRequest( + &streamservice.AppendRecordsRequest{ + Producer: retry, + Sequence: 1, + Records: records("b"), + WithdrawRun: "run", + }, + ) require.NoError(t, err) _, err = h.append(workflowProducer(9), 1, "c") require.ErrorContains(t, err, ReasonFenced) @@ -508,8 +768,8 @@ func TestErrorsCarryTheReason(t *testing.T) { require.Equal(t, ReasonFenced, details[0].(*streamservice.StreamErrorDetails).GetReason()) } -// A reader's floor keeps its Records past the Retention and the cap, and the Records below it -// trim. When the reader goes, the Retention trims the rest. +// A reader's floor keeps its Records past the Retention and the cap, and the Records below it are +// trimmed. When the floor is dropped, the Retention trims the rest. func TestReaderFloorKeepsItsRecords(t *testing.T) { h := newHarness(t) h.limits.MaxStoredBytes = 200 @@ -522,9 +782,14 @@ func TestReaderFloorKeepsItsRecords(t *testing.T) { b, err := h.append(activityProducer(1), 2, "b") require.NoError(t, err) setReaders := func(fn func(*Stream, chasm.MutableContext) error) { - _, _, err := chasm.UpdateComponent(h.ctx, h.ref(), func(s *Stream, mctx chasm.MutableContext, _ struct{}) (struct{}, error) { - return struct{}{}, fn(s, mctx) - }, struct{}{}) + _, _, err := chasm.UpdateComponent( + h.ctx, + h.ref(), + func(s *Stream, mctx chasm.MutableContext, _ struct{}) (struct{}, error) { + return struct{}{}, fn(s, mctx) + }, + struct{}{}, + ) require.NoError(t, err) } setReaders(func(s *Stream, mctx chasm.MutableContext) error { @@ -533,9 +798,13 @@ func TestReaderFloorKeepsItsRecords(t *testing.T) { b.GetFirstOffset(), h.limits) }) - big := &streamservice.AppendRecordsRequest{Producer: activityProducer(1), Sequence: 3, Records: []*streamservice.AppendRecord{ - {Topic: "c", Body: &commonpb.Payload{Data: make([]byte, 300)}}, - }} + big := &streamservice.AppendRecordsRequest{ + Producer: activityProducer(1), + Sequence: 3, + Records: []*streamservice.AppendRecord{ + {Topic: "c", Body: &commonpb.Payload{Data: make([]byte, 300)}}, + }, + } _, err = h.appendRequest(big) require.ErrorContains(t, err, ReasonTooLarge) @@ -553,9 +822,75 @@ func TestReaderFloorKeepsItsRecords(t *testing.T) { require.ErrorContains(t, err, ReasonExpired) } -// localActivityWrite is a write of the Local Activity call of the run, dispatched by task -// started, with the run's last completed task (0 before the first), as the crate's S110 test has. -func localActivityWrite(call string, started, completed int64, topic string) *streamservice.AppendRecordsRequest { +// The readers check drops the wake of a run that closed, so dead wakes don't fill the cap of +// waiting reads until their wait ends. +func TestReadersCheckDropsTheWakeOfAClosedRun(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + result, err := chasm.UpdateWithStartExecution(h.ctx, h.key, StartForRead, (*Stream).ReadOrPark, + ReadInput{ + Request: &streamservice.ReadRecordsRequest{WakeOnly: true}, + RequestID: "wake", + Callback: &commonpb.Callback{Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{Url: chasm.NexusCompletionHandlerURL}, + }}, + Reader: &ReaderRun{WorkflowID: "wf", RunID: "run-1"}, + Limits: h.limits, + }, + chasm.WithBusinessIDPolicy( + chasm.BusinessIDReusePolicyRejectDuplicate, chasm.BusinessIDConflictPolicyUseExisting), + ) + require.NoError(t, err) + require.Nil(t, result.UpdateOutput) + require.False(t, h.parked("wake").GetResolved()) + + _, _, err = chasm.UpdateComponent(h.ctx, h.ref(), + func(s *Stream, mctx chasm.MutableContext, _ struct{}) (struct{}, error) { + s.dropReaders(mctx, []string{"run-1"}) + return struct{}{}, nil + }, struct{}{}) + require.NoError(t, err) + read := h.parked("wake") + require.True(t, read.GetResolved()) + require.NotNil(t, read.GetFailure().GetCanceledFailureInfo()) +} + +// A client's Stream keeps the floor of a Workflow reader too, so the size cap can't drop what the +// reader's replay reads again. The append that would drop it fails instead. +func TestReaderFloorKeepsTheRecordsOfAClientStream(t *testing.T) { + h := newHarness(t) + // The cap takes the large batch alone, but not with the batch of "a". + h.limits.MaxStoredBytes = 270 + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + a, err := h.append(clientProducer(), 1, "a") + require.NoError(t, err) + _, _, err = chasm.UpdateComponent(h.ctx, h.ref(), + func(s *Stream, mctx chasm.MutableContext, _ struct{}) (struct{}, error) { + return struct{}{}, s.AddReader(mctx, ReaderRun{WorkflowID: "wf", RunID: "run-1"}, + a.GetFirstOffset(), h.limits) + }, struct{}{}) + require.NoError(t, err) + + _, err = h.appendRequest(&streamservice.AppendRecordsRequest{ + Producer: clientProducer(), Sequence: 2, Records: []*streamservice.AppendRecord{ + {Topic: "big", Body: &commonpb.Payload{Data: make([]byte, 200)}}, + }, + }) + require.ErrorContains(t, err, ReasonTooLarge) + page, err := h.read(&streamservice.ReadRecordsRequest{FromOffset: a.GetFirstOffset()}) + require.NoError(t, err) + require.Equal(t, []string{"a"}, topics(page)) +} + +// localActivityWrite is a write of the run's Local Activity call from the task that started at +// started. It has the run's last completed task, or 0 before the first, as the crate's tests do. +func localActivityWrite( + call string, + started, completed int64, + topic string, +) *streamservice.AppendRecordsRequest { return &streamservice.AppendRecordsRequest{ Producer: &streamservice.Producer{ Kind: streamservice.PRODUCER_KIND_LOCAL_ACTIVITY, @@ -590,9 +925,13 @@ func TestLocalActivityOfATaskThatNeverCompletedIsWithdrawnByTheNextWrite(t *test require.Equal(t, []string{"done", "orphan", "superseded", "other"}, topics(read)) require.Equal(t, read.GetRecords()[3].GetOffset(), other.GetFirstOffset()) require.Equal(t, &streamservice.Supersession{ - ProducerKind: streamservice.PRODUCER_KIND_WORKFLOW, - ProducerId: "wf:run", - Epoch: &streamservice.ProducerEpoch{TaskStartedEventId: 4, TaskAttempt: 1, TaskStartedTime: timestamppb.New(time.Unix(104, 0))}, + ProducerKind: streamservice.PRODUCER_KIND_WORKFLOW, + ProducerId: "wf:run", + Epoch: &streamservice.ProducerEpoch{ + TaskStartedEventId: 4, + TaskAttempt: 1, + TaskStartedTime: timestamppb.New(time.Unix(104, 0)), + }, PreviousCompletedTask: &streamservice.WorkflowTaskEpoch{StartedEventId: 2}, RunId: "run", }, read.GetRecords()[2].GetSupersession()) @@ -677,23 +1016,19 @@ func TestCallerCantCloseAsTerminated(t *testing.T) { require.Equal(t, streamservice.CLOSE_STATUS_COMPLETED, read.GetRecords()[0].GetCloseStatus()) } -// A wake_only read parks for hours, so an idle Workflow reader adds no History each minute. Any -// other parked read still answers an empty page after 60 s. +// A wake_only read parks for hours, so an idle Workflow reader adds no History each minute. func TestWakeParksPastTheLongPollCap(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) require.NoError(t, err) - for id, wakeOnly := range map[string]bool{"wake": true, "page": false} { - resp, err := h.readOrPark(id, &streamservice.ReadRecordsRequest{ - WakeOnly: wakeOnly, Wait: durationpb.New(2 * time.Hour), - }) - require.NoError(t, err) - require.Nil(t, resp) - } + resp, err := h.readOrPark("wake", &streamservice.ReadRecordsRequest{ + WakeOnly: true, Wait: durationpb.New(2 * time.Hour), + }) + require.NoError(t, err) + require.Nil(t, resp) h.advance(2 * time.Minute) require.False(t, h.parked("wake").GetResolved()) - require.True(t, h.parked("page").GetResolved()) _, err = h.append(activityProducer(1), 1, "a") require.NoError(t, err) @@ -701,8 +1036,28 @@ func TestWakeParksPastTheLongPollCap(t *testing.T) { require.Empty(t, h.parked("wake").GetResponse().GetRecords()) } -// A page stops before the Record that would take it past max_bytes, and a larger Record comes -// alone, so a reader with a small cap still moves on. +// Only a wake parks. Any other read answers what the Stream has at once, so no parked read +// keeps a page, and one append that wakes many reads doesn't copy its batch into each. +func TestOnlyAWakeParks(t *testing.T) { + h := newHarness(t) + _, err := h.readOrPark("pending", &streamservice.ReadRecordsRequest{ + CreateWait: durationpb.New(time.Minute), + }) + require.ErrorContains(t, err, ReasonNotFound) + + _, err = h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + resp, err := h.readOrPark("page", &streamservice.ReadRecordsRequest{ + Start: streamservice.READ_START_END, Wait: durationpb.New(time.Minute), + }) + require.NoError(t, err) + require.NotNil(t, resp) + require.Empty(t, resp.GetRecords()) + require.Nil(t, h.parked("page")) +} + +// A page stops before the Record that would take it past max_bytes, and a larger Record is alone +// on its page, so a reader with a small cap still makes progress. func TestPageStopsAtMaxBytes(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) @@ -726,6 +1081,45 @@ func TestPageStopsAtMaxBytes(t *testing.T) { require.Equal(t, []string{"big"}, topics(second)) } +// skipped is one batch with more entries of another Topic than one page scans, then a match. +func skipped() []string { + out := make([]string, 0, maxScan+2) + for range maxScan + 1 { + out = append(out, "other") + } + return append(out, "turns") +} + +// A filtered page scans past the scan cap while it has no Record, so a read with no wait doesn't +// take an empty page for the end of the Stream. +func TestFilteredPageScansPastTheCapToAMatch(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + _, err = h.append(activityProducer(1), 1, skipped()...) + require.NoError(t, err) + + page, err := h.read(&streamservice.ReadRecordsRequest{Topics: []string{"turns"}}) + require.NoError(t, err) + require.Equal(t, []string{"turns"}, topics(page)) +} + +// A parked read's look scans past the scan cap too, so it wakes with no later append. +func TestParkedReadWakesForAMatchPastTheCap(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{ + Topics: []string{"turns"}, WakeOnly: true, + }) + require.NoError(t, err) + require.Nil(t, resp) + + _, err = h.append(activityProducer(1), 1, skipped()...) + require.NoError(t, err) + require.True(t, h.parked("read-1").GetResolved()) +} + // A last read gives the newest Records that pass the filter, oldest first, and leaves out the // Records of a withdrawn Workflow Task. func TestLastReadsTheNewestRecords(t *testing.T) { @@ -834,7 +1228,7 @@ func TestReplayOfAPagePastTheCloseGetsTheClose(t *testing.T) { } // A client's higher epoch fences its lower epochs but withdraws none of their Records, since a -// client process writes on and doesn't redo the work. +// client process doesn't redo its writes. func TestClientEpochFencesButWithdrawsNothing(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) @@ -877,8 +1271,8 @@ func TestRecordsOfOneAppendHaveConsecutiveOffsets(t *testing.T) { require.Equal(t, resp.GetFirstOffset()+2, resp.GetLastOffset()) } -// At the Producer cap a Local Activity's entry goes first, since a call writes only while it -// runs. A client's older entry keeps its dedupe state, so its resend still goes in once. +// At the Producer cap a Local Activity's entry is dropped first, since a call writes only while it +// runs. A client's older entry keeps its dedupe state, so its resend is still written once. func TestLocalActivityProducerGoesFirstAtTheCap(t *testing.T) { h := newHarness(t) h.limits.MaxProducers = 2 diff --git a/chasm/lib/stream/tasks.go b/chasm/lib/stream/tasks.go index a2399065004..268420ea442 100644 --- a/chasm/lib/stream/tasks.go +++ b/chasm/lib/stream/tasks.go @@ -20,7 +20,8 @@ func (h *timerTaskHandler) Validate( _ chasm.TaskInvocation, task *streampb.StreamTimerTask, ) (bool, error) { - return s.GetTimerDeadline() != nil && s.GetTimerDeadline().AsTime().Equal(task.GetDeadline().AsTime()), nil + return s.GetTimerDeadline() != nil && + s.GetTimerDeadline().AsTime().Equal(task.GetDeadline().AsTime()), nil } func (h *timerTaskHandler) Execute( diff --git a/chasm/lib/stream/testdata/epochs.json b/chasm/lib/stream/testdata/epochs.json new file mode 100644 index 00000000000..ce1fe07355c --- /dev/null +++ b/chasm/lib/stream/testdata/epochs.json @@ -0,0 +1,55 @@ +[ + { + "kind": "workflow", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 2, + "task_started_time": [ + 1760000000, + 5 + ] + }, + "key": "0000000000000008.00000002.0000000068e77800.00000005.0000000000000000.00000000.00000000", + "task_key": "0000000000000008.00000002.0000000068e77800.00000005" + }, + { + "kind": "local_activity", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 1760000000, + 5 + ], + "attempt": 3 + }, + "key": "0000000000000008.00000001.0000000068e77800.00000005.0000000000000000.00000000.00000003", + "task_key": "0000000000000008.00000001.0000000068e77800.00000005" + }, + { + "kind": "activity", + "epoch": { + "attempt_scheduled_time": [ + 1760000001, + 999999999 + ], + "attempt": 2 + }, + "key": "0000000000000000.00000000.0000000000000000.00000000.0000000068e77801.3b9ac9ff.00000002", + "task_key": "0000000000000000.00000000.0000000000000000.00000000" + }, + { + "kind": "client", + "epoch": { + "attempt": 1 + }, + "key": "0000000000000000.00000000.0000000000000000.00000000.0000000000000000.00000000.00000001", + "task_key": "0000000000000000.00000000.0000000000000000.00000000" + }, + { + "kind": "empty", + "epoch": {}, + "key": "0000000000000000.00000000.0000000000000000.00000000.0000000000000000.00000000.00000000", + "task_key": "0000000000000000.00000000.0000000000000000.00000000" + } +] diff --git a/chasm/lib/stream/testdata/withdrawals.json b/chasm/lib/stream/testdata/withdrawals.json new file mode 100644 index 00000000000..128c21425f1 --- /dev/null +++ b/chasm/lib/stream/testdata/withdrawals.json @@ -0,0 +1,1354 @@ +[ + { + "name": "e1 B at the failed task 5 goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e1 a Record at the completed task 8 stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e1 a Record before c stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e2 B goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 2, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e2 an older completed Record stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 2, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 2, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e3 D at a lower attempt of t goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 3, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 5, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e3 a Record of t itself stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 3, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 5, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 3, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e4 U at an earlier started time goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 9, + "task_attempt": 1, + "task_started_time": [ + 200, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 6, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 9, + "task_attempt": 1, + "task_started_time": [ + 150, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e5 D between c and t goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 5, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e5 a Record at c stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 5, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "c attempt 0, an earlier attempt at c goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 0, + "started_time": [ + 300, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 200, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "c attempt 0, c itself stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 0, + "started_time": [ + 300, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 2, + "task_started_time": [ + 300, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e1 a Local Activity of the failed task goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 2 + } + } + }, + "withdraws": true + }, + { + "name": "another run's Record stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r9", + "run_id": "r9", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e6 r1's Record goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "withdraw_run": "r1" + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 20, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e6 r1's Activity Record goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "withdraw_run": "r1" + }, + "record": { + "producer": { + "kind": "activity", + "id": "act:r1:a:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "e6 r2's own earlier Record of t stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "withdraw_run": "r1" + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "an Activity retry withdraws attempt 1", + "supersession": { + "producer_kind": "activity", + "producer_id": "act:r1:a:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 130, + 0 + ], + "attempt": 2 + } + }, + "record": { + "producer": { + "kind": "activity", + "id": "act:r1:a:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "an Activity retry keeps another Activity", + "supersession": { + "producer_kind": "activity", + "producer_id": "act:r1:a:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 130, + 0 + ], + "attempt": 2 + } + }, + "record": { + "producer": { + "kind": "activity", + "id": "act:r1:b:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a client's newer epoch keeps its earlier Records", + "supersession": { + "producer_kind": "client", + "producer_id": "client:ingest", + "epoch": { + "attempt": 2 + } + }, + "record": { + "producer": { + "kind": "client", + "id": "client:ingest", + "epoch": { + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "no epoch, r1's Record goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "withdraw_run": "r1", + "failed_tasks": [ + 5 + ] + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "no epoch, r1's Local Activity Record goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "withdraw_run": "r1", + "failed_tasks": [ + 5 + ] + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "no epoch, r2's Local Activity Record at a failed task stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "withdraw_run": "r1", + "failed_tasks": [ + 5 + ] + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r2:k", + "run_id": "r2", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "no epoch, r2's Record stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "withdraw_run": "r1", + "failed_tasks": [ + 5 + ] + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity's newer epoch withdraws a lower attempt", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 2 + } + } + }, + "withdraws": true + }, + { + "name": "a Local Activity's newer epoch withdraws an earlier task", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 4 + } + } + }, + "withdraws": true + }, + { + "name": "a Local Activity's newer epoch keeps its own epoch", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity's newer epoch keeps another call", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:j", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity's item keeps the run's Workflow Record", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity before c stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity at c stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity between c and t goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "a Local Activity of a lower attempt of t goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "a Local Activity of t itself stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity of another run stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r9:k", + "run_id": "r9", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity Record with no epoch stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1" + } + }, + "withdraws": false + } +] diff --git a/chasm/lib/stream/vectors_test.go b/chasm/lib/stream/vectors_test.go new file mode 100644 index 00000000000..b2ba1ecb619 --- /dev/null +++ b/chasm/lib/stream/vectors_test.go @@ -0,0 +1,164 @@ +package stream + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/streamservice/v1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// The files in testdata are copies of the crate's shared vectors, at moetemp/sdk-streams +// crates/streams/testdata, so the server's Producer rules and the Stores agree. + +type vectorTime [2]int64 + +func (t *vectorTime) proto() *timestamppb.Timestamp { + if t == nil { + return nil + } + return ×tamppb.Timestamp{Seconds: t[0], Nanos: int32(t[1])} +} + +type vectorEpoch struct { + TaskStartedEventID int64 `json:"task_started_event_id"` + TaskAttempt int32 `json:"task_attempt"` + TaskStartedTime *vectorTime `json:"task_started_time"` + AttemptScheduledTime *vectorTime `json:"attempt_scheduled_time"` + Attempt int32 `json:"attempt"` +} + +func (e *vectorEpoch) proto() *streamservice.ProducerEpoch { + if e == nil { + return nil + } + return &streamservice.ProducerEpoch{ + TaskStartedEventId: e.TaskStartedEventID, + TaskAttempt: e.TaskAttempt, + TaskStartedTime: e.TaskStartedTime.proto(), + AttemptScheduledTime: e.AttemptScheduledTime.proto(), + Attempt: e.Attempt, + } +} + +var vectorKinds = map[string]streamservice.ProducerKind{ + "workflow": streamservice.PRODUCER_KIND_WORKFLOW, + "local_activity": streamservice.PRODUCER_KIND_LOCAL_ACTIVITY, + "activity": streamservice.PRODUCER_KIND_ACTIVITY, + "standalone_activity": streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY, + "client": streamservice.PRODUCER_KIND_CLIENT, +} + +func readVectors(t *testing.T, name string, into any) { + data, err := os.ReadFile("testdata/" + name) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, into)) +} + +// vectorWithdraws applies a Superseded Record to a Record with the server's rules. A Workflow item +// uses the task rule of withdrawTasks, and the other kinds that withdraw use CompareEpochs. +func vectorWithdraws(s *streamservice.Supersession, p *streamservice.Producer) bool { + if s.GetWithdrawRun() != "" && p.GetRunId() == s.GetWithdrawRun() { + return true + } + if s.GetEpoch() == nil || p.GetEpoch() == nil { + return false + } + if s.GetProducerKind() != streamservice.PRODUCER_KIND_WORKFLOW { + return withdrawsEpochs(s.GetProducerKind()) && p.GetKind() == s.GetProducerKind() && + p.GetId() == s.GetProducerId() && CompareEpochs(p.GetEpoch(), s.GetEpoch()) < 0 + } + switch p.GetKind() { + case streamservice.PRODUCER_KIND_WORKFLOW: + if p.GetId() != s.GetProducerId() { + return false + } + case streamservice.PRODUCER_KIND_LOCAL_ACTIVITY: + if p.GetRunId() != s.GetRunId() { + return false + } + default: + return false + } + index := []*streamservice.WorkflowTaskEpoch{taskOf(p.GetEpoch())} + split := withdrawTasks(index, taskOf(s.GetEpoch()), s.GetPreviousCompletedTask(), + s.GetFailedTasks()) + return len(split.withdrawn) > 0 +} + +func TestWithdrawalsMatchTheCrateVectors(t *testing.T) { + var vectors []struct { + Name string `json:"name"` + Supersession struct { + ProducerKind string `json:"producer_kind"` + ProducerID string `json:"producer_id"` + RunID string `json:"run_id"` + Epoch *vectorEpoch `json:"epoch"` + FailedTasks []int64 `json:"failed_tasks"` + WithdrawRun string `json:"withdraw_run"` + Completed *struct { + StartedEventID int64 `json:"started_event_id"` + Attempt int32 `json:"attempt"` + StartedTime *vectorTime `json:"started_time"` + } `json:"previous_completed_task"` + } `json:"supersession"` + Record struct { + Producer struct { + Kind string `json:"kind"` + ID string `json:"id"` + RunID string `json:"run_id"` + Epoch *vectorEpoch `json:"epoch"` + } `json:"producer"` + } `json:"record"` + Withdraws bool `json:"withdraws"` + } + readVectors(t, "withdrawals.json", &vectors) + require.NotEmpty(t, vectors) + for _, v := range vectors { + s := &streamservice.Supersession{ + ProducerKind: vectorKinds[v.Supersession.ProducerKind], + ProducerId: v.Supersession.ProducerID, + RunId: v.Supersession.RunID, + Epoch: v.Supersession.Epoch.proto(), + FailedTasks: v.Supersession.FailedTasks, + WithdrawRun: v.Supersession.WithdrawRun, + } + if c := v.Supersession.Completed; c != nil { + s.PreviousCompletedTask = &streamservice.WorkflowTaskEpoch{ + StartedEventId: c.StartedEventID, + Attempt: c.Attempt, + StartedTime: c.StartedTime.proto(), + } + } + p := v.Record.Producer + producer := &streamservice.Producer{ + Kind: vectorKinds[p.Kind], Id: p.ID, RunId: p.RunID, Epoch: p.Epoch.proto(), + } + require.Equal(t, v.Withdraws, vectorWithdraws(s, producer), v.Name) + } +} + +// The crate's epoch keys sort as its epochs compare, so CompareEpochs must give the same order. +func TestCompareEpochsMatchesTheCrateKeys(t *testing.T) { + var vectors []struct { + Epoch vectorEpoch `json:"epoch"` + Key string `json:"key"` + } + readVectors(t, "epochs.json", &vectors) + require.NotEmpty(t, vectors) + for _, a := range vectors { + for _, b := range vectors { + require.Equal( + t, + strings.Compare(a.Key, b.Key), + CompareEpochs(a.Epoch.proto(), b.Epoch.proto()), + "%s against %s", + a.Key, + b.Key, + ) + } + } +} diff --git a/chasm/lib/workflow/workflow.go b/chasm/lib/workflow/workflow.go index 3f52aa39f18..26bc7b7bc04 100644 --- a/chasm/lib/workflow/workflow.go +++ b/chasm/lib/workflow/workflow.go @@ -52,7 +52,7 @@ type Workflow struct { Updates chasm.Map[string, *WorkflowUpdate] // Streams are the native Streams of the run chain, keyed by Stream id. They nest in the - // chain's first run, since a run's tree doesn't move to the next run. + // chain's first run, since each run of a chain has its own tree. Streams chasm.Map[string, *stream.Stream] } @@ -118,8 +118,8 @@ func (w *Workflow) CheckWriter(chasm.Context, *streamservice.Producer, []byte) e return nil } -// HasChainStreams says whether the final close of this run may have Streams to close: its own, -// or, for a later run of a chain, the ones nested in the chain's first run. +// HasChainStreams says whether the final close of this run may have Streams to close. Those are +// its own, or, for a later run of a chain, the ones nested in the chain's first run. func (w *Workflow) HasChainStreams(ctx chasm.Context, firstRunID string) bool { if len(w.Streams) > 0 { return true @@ -133,13 +133,7 @@ func (w *Workflow) HasChainStreams(ctx chasm.Context, firstRunID string) bool { // namespace retention that would delete it. A chain that continues as new keeps writing to the // Streams of its first run. func (w *Workflow) HoldsStreams(ctx chasm.Context) bool { - for _, field := range w.Streams { - if s, ok := field.TryGet(ctx); ok && s.Phase != streampb.STREAM_PHASE_EXPIRED && - s.Phase != streampb.STREAM_PHASE_ABANDONED { - return true - } - } - return false + return stream.HoldsStreams(ctx, w.Streams) } // EndChainStreams closes the chain's close-with-run Streams with this run's close event. The @@ -167,7 +161,7 @@ func (w *Workflow) EndChainStreams( // DropActivityProducers drops a closed Activity's Producer entries from the Streams of this run. // After continue-as-new the Streams are in the chain's first run, another execution, and keep -// the entries until the Producer cap. +// the entries until the Producer cap drops them. func (w *Workflow) DropActivityProducers(ctx chasm.MutableContext, activityID string) { runID := ctx.ExecutionKey().RunID for _, field := range w.Streams { diff --git a/common/rpc/interceptor/namespace_validator.go b/common/rpc/interceptor/namespace_validator.go index 50dc45e2cc4..9b8ed698ba7 100644 --- a/common/rpc/interceptor/namespace_validator.go +++ b/common/rpc/interceptor/namespace_validator.go @@ -425,8 +425,8 @@ func (ni *NamespaceValidatorInterceptor) extractNamespaceFromTaskToken(req any) return ni.namespaceRegistry.GetNamespaceByID(namespaceID) } -// optionalTaskToken says whether a request carries a task token only from some callers. A Stream -// write carries one only from an Activity, and other callers set the namespace. +// optionalTaskToken says whether a request has a task token only from some callers. A Stream +// write has one only from an Activity, and other callers set the namespace. func optionalTaskToken(req any) bool { switch req.(type) { case *streamservice.CreateStreamRequest, diff --git a/go.sum b/go.sum index 079bfbc0bfc..c80ec64a793 100644 --- a/go.sum +++ b/go.sum @@ -481,8 +481,6 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= -go.temporal.io/api v1.63.8-0.20261014221631-25d6833c5701 h1:4FS8eouE2ml0soTg/pht5tk/tBT5OpI64DGHXpJ0uqo= -go.temporal.io/api v1.63.8-0.20261014221631-25d6833c5701/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= go.temporal.io/auto-scaled-workers v0.0.0-20261013231030-67ffdef6330d h1:cKo/dGcLpFAZXKxIAR2sg9hI+h8/8Z0tNHxXA7LCRVI= go.temporal.io/auto-scaled-workers v0.0.0-20261013231030-67ffdef6330d/go.mod h1:DpnPBMkwtk/08wMxwR3u3mIOshsgK/V7KKLh1QzNdjw= go.temporal.io/sdk v1.49.0 h1:CtGI0BUe/SCo3eoqTwuWWtXKueii9GBVus7KrKKH1Vo= diff --git a/service/frontend/fx.go b/service/frontend/fx.go index c7aaa515b63..fe479fca24a 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -437,7 +437,8 @@ func RedirectionInterceptorProvider( timeSource, clusterMetadata, ).WithRedirectResponses(chasmstream.RedirectableMethods()). - WithAdditionalAllowedMethods(slices.Collect(maps.Keys(chasmstream.RedirectableMethods()))...) + WithAdditionalAllowedMethods( + slices.Collect(maps.Keys(chasmstream.RedirectableMethods()))...) } func BusinessIDInterceptorProvider( diff --git a/service/frontend/nexus_direct.go b/service/frontend/nexus_direct.go index bf559cc31d2..8769eec1e6e 100644 --- a/service/frontend/nexus_direct.go +++ b/service/frontend/nexus_direct.go @@ -17,8 +17,8 @@ import ( ) // StartNexusOperation runs one operation of the System Nexus endpoint for the caller's namespace, -// as a direct call (temporalio/api#720). It creates no caller-side state and no History event, -// so a Workflow can read through it with nothing in its History. +// as a direct call. It creates no caller-side state and no History event, so a Workflow can read +// through it with nothing in its History. func (wh *WorkflowHandler) StartNexusOperation( ctx context.Context, req *workflowservice.StartNexusOperationRequest, @@ -27,7 +27,9 @@ func (wh *WorkflowHandler) StartNexusOperation( return nil, serviceerror.NewUnimplemented("Direct Nexus calls are disabled") } if req.GetEndpoint() != commonnexus.SystemEndpoint { - return nil, serviceerror.NewUnimplemented("Direct Nexus calls serve only the __temporal_system endpoint") + return nil, serviceerror.NewUnimplemented( + "Direct Nexus calls serve only the __temporal_system endpoint", + ) } ns, err := wh.namespaceRegistry.GetNamespace(namespace.Name(req.GetNamespace())) if err != nil { @@ -37,25 +39,33 @@ func (wh *WorkflowHandler) StartNexusOperation( if requestID == "" { requestID = uuid.NewString() } - processed, err := wh.registry.NexusEndpointProcessor.ProcessInput(chasm.NexusOperationProcessorContext{ - Namespace: ns, - RequestID: requestID, - ReserializeInputPayload: true, - }, req.GetService(), req.GetOperation(), req.GetInput()) + processed, err := wh.registry.NexusEndpointProcessor.ProcessInput( + chasm.NexusOperationProcessorContext{ + Namespace: ns, + RequestID: requestID, + ReserializeInputPayload: true, + }, + req.GetService(), + req.GetOperation(), + req.GetInput(), + ) if err != nil { return nil, err } - resp, err := wh.historyClient.StartNexusOperation(ctx, &historyservice.StartNexusOperationRequest{ - NamespaceId: ns.ID().String(), - ShardId: processed.RoutingKey.ShardID(wh.config.NumHistoryShards), - Request: &nexuspb.StartOperationRequest{ - Service: req.GetService(), - Operation: req.GetOperation(), - Payload: processed.ReserializedInputPayload, - RequestId: requestID, - Links: workflowEventLinks(req.GetLinks(), requestID), + resp, err := wh.historyClient.StartNexusOperation( + ctx, + &historyservice.StartNexusOperationRequest{ + NamespaceId: ns.ID().String(), + ShardId: processed.RoutingKey.ShardID(wh.config.NumHistoryShards), + Request: &nexuspb.StartOperationRequest{ + Service: req.GetService(), + Operation: req.GetOperation(), + Payload: processed.ReserializedInputPayload, + RequestId: requestID, + Links: workflowEventLinks(req.GetLinks(), requestID), + }, }, - }) + ) if err != nil { return nil, err } @@ -63,7 +73,9 @@ func (wh *WorkflowHandler) StartNexusOperation( case *nexuspb.StartOperationResponse_SyncSuccess: return &workflowservice.StartNexusOperationResponse{ Variant: &workflowservice.StartNexusOperationResponse_SyncSuccess_{ - SyncSuccess: &workflowservice.StartNexusOperationResponse_SyncSuccess{Payload: v.SyncSuccess.GetPayload()}, + SyncSuccess: &workflowservice.StartNexusOperationResponse_SyncSuccess{ + Payload: v.SyncSuccess.GetPayload(), + }, }, Links: v.SyncSuccess.GetLinks(), }, nil @@ -79,7 +91,9 @@ func (wh *WorkflowHandler) StartNexusOperation( case *nexuspb.StartOperationResponse_Failure: return &workflowservice.StartNexusOperationResponse{ Variant: &workflowservice.StartNexusOperationResponse_OperationFailure_{ - OperationFailure: &workflowservice.StartNexusOperationResponse_OperationFailure{Failure: v.Failure}, + OperationFailure: &workflowservice.StartNexusOperationResponse_OperationFailure{ + Failure: v.Failure, + }, }, }, nil default: @@ -87,9 +101,8 @@ func (wh *WorkflowHandler) StartNexusOperation( } } -// workflowEventLinks passes on the caller's Workflow links, so an operation knows the reading run. -// A direct call names no event of its caller, and the Nexus form of a link needs a reference, so a -// link with none refers to the call's request id. +// workflowEventLinks gives an operation the caller's Workflow links, so it knows the reading run. +// The Nexus form of a link needs a reference, so a link with none refers to the call's request id. func workflowEventLinks(links []*commonpb.Link, requestID string) []*nexuspb.Link { var out []nexus.Link for _, link := range links { diff --git a/service/history/timer_queue_task_executor_base.go b/service/history/timer_queue_task_executor_base.go index 01e1608c5b7..9655b90c61c 100644 --- a/service/history/timer_queue_task_executor_base.go +++ b/service/history/timer_queue_task_executor_base.go @@ -11,6 +11,7 @@ import ( enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" + chasmstream "go.temporal.io/server/chasm/lib/stream" "go.temporal.io/server/common/locks" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" @@ -137,23 +138,27 @@ func (t *timerQueueTaskExecutorBase) executeDeleteHistoryEventTask( return err } - if task.ArchetypeID == chasm.WorkflowArchetypeID && mutableState.ChasmEnabled() { - wf, chasmCtx, err := mutableState.ChasmWorkflowComponentReadOnly(ctx) - if err != nil { - return err - } - // The first run of a chain keeps the chain's Streams, which the chain may still write - // after this run closed. The run stays one more retention, until its Streams expire. - if wf.HoldsStreams(chasmCtx) { - mutableState.AddTasks(&tasks.DeleteHistoryEventTask{ - WorkflowKey: task.WorkflowKey, - VisibilityTimestamp: t.shardContext.GetTimeSource().Now().Add(mutableState.GetNamespaceEntry().Retention()), - Version: task.Version, - BranchToken: task.BranchToken, - ArchetypeID: task.ArchetypeID, - }) - return weContext.SubmitClosedWorkflowSnapshot(ctx, t.shardContext, historyi.TransactionPolicyActive) + holds, err := holdsStreams(ctx, task.ArchetypeID, mutableState) + if err != nil { + return err + } + // A chain writes its first run's Streams after that run closed, and a closed Standalone + // Activity keeps its Streams for their Retention. So the execution waits one more retention. + if holds { + mutableState.AddTasks(&tasks.DeleteHistoryEventTask{ + WorkflowKey: task.WorkflowKey, + VisibilityTimestamp: t.shardContext.GetTimeSource().Now(). + Add(mutableState.GetNamespaceEntry().Retention()), + Version: task.Version, + BranchToken: task.BranchToken, + ArchetypeID: task.ArchetypeID, + }) + // A standby cluster writes the closed execution as its replication does. + policy := historyi.TransactionPolicyActive + if !t.isActive { + policy = historyi.TransactionPolicyPassive } + return weContext.SubmitClosedWorkflowSnapshot(ctx, t.shardContext, policy) } return t.deleteManager.DeleteWorkflowExecutionByRetention( @@ -166,6 +171,36 @@ func (t *timerQueueTaskExecutorBase) executeDeleteHistoryEventTask( ) } +// streamHost is the root component of an execution that nests Streams. +type streamHost interface { + HoldsStreams(chasm.Context) bool +} + +// holdsStreams says whether a closed execution nests a Stream that is still readable. Only a +// Workflow and a Standalone Activity nest Streams. +func holdsStreams( + ctx context.Context, + archetypeID chasm.ArchetypeID, + mutableState historyi.MutableState, +) (bool, error) { + nests := archetypeID == chasm.WorkflowArchetypeID || + archetypeID == chasmstream.ActivityArchetypeID + if !nests || !mutableState.ChasmEnabled() { + return false, nil + } + tree, ok := mutableState.ChasmTree().(*chasm.Node) + if !ok { + return false, nil + } + chasmCtx := chasm.NewContext(ctx, tree) + root, err := tree.ComponentByPath(chasmCtx, nil) + if err != nil { + return false, err + } + host, ok := root.(streamHost) + return ok && host.HoldsStreams(chasmCtx), nil +} + func (t *timerQueueTaskExecutorBase) deleteHistoryBranch( ctx context.Context, branchToken []byte, diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index a0aa8ed7ff6..305691bd742 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -7528,8 +7528,9 @@ func (ms *MutableStateImpl) processCloseCallbacksChasm(closeEvent *historypb.His } firstRunID := ms.GetExecutionInfo().GetFirstExecutionRunId() - // A cron run's end isn't the chain's end. The next run writes on to the chain's Streams. - chainEnds := ms.executionInfo.NewExecutionRunId == "" + // A cron run's end isn't the chain's end, nor is the end of a run that a reset terminates. The + // next run or the reset run writes on to the chain's Streams. + chainEnds := ms.executionInfo.NewExecutionRunId == "" && !ms.executionInfo.GetWorkflowWasReset() hasStreams := chainEnds && wf.HasChainStreams(readCtx, firstRunID) // Return early if there are no chasm callbacks to process. diff --git a/tests/stream_test.go b/tests/stream_test.go index babba2117a4..289c1b26330 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -1,6 +1,7 @@ package tests import ( + "errors" "testing" "time" @@ -16,6 +17,8 @@ import ( taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/sdk/client" + "go.temporal.io/server/api/adminservice/v1" + "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/stream" "go.temporal.io/server/common/dynamicconfig" commonnexus "go.temporal.io/server/common/nexus" @@ -63,22 +66,28 @@ type streamCaller struct { func (s *StreamTestSuite) newCaller(env *testcore.TestEnv) *streamCaller { c := &streamCaller{s: s, env: env, taskQueue: testcore.RandomizeStr(s.T().Name())} - run, err := env.SdkClient().ExecuteWorkflow(s.Context(), client.StartWorkflowOptions{TaskQueue: c.taskQueue}, "stream-caller") + options := client.StartWorkflowOptions{TaskQueue: c.taskQueue} + run, err := env.SdkClient().ExecuteWorkflow(s.Context(), options, "stream-caller") s.NoError(err) c.run = run s.T().Cleanup(func() { - _ = env.SdkClient().TerminateWorkflow(s.Context(), run.GetID(), run.GetRunID(), "test cleanup") + _ = env.SdkClient(). + TerminateWorkflow(s.Context(), run.GetID(), run.GetRunID(), "test cleanup") }) c.poll() return c } func (c *streamCaller) poll() { - resp, err := c.env.FrontendClient().PollWorkflowTaskQueue(c.s.Context(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: c.env.Namespace().String(), - TaskQueue: &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - Identity: "test", - }) + resp, err := c.env.FrontendClient(). + PollWorkflowTaskQueue(c.s.Context(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: c.env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{ + Name: c.taskQueue, + Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + }, + Identity: "test", + }) c.s.NoError(err) c.s.NotEmpty(resp.GetTaskToken()) c.token = resp.GetTaskToken() @@ -92,17 +101,7 @@ func (c *streamCaller) schedule(ops ...streamOp) []int64 { last := c.history[len(c.history)-1].GetEventId() commands := make([]*commandpb.Command, 0, len(ops)) for _, op := range ops { - commands = append(commands, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ - ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ - Endpoint: commonnexus.SystemEndpoint, - Service: stream.ServiceName, - Operation: op.name, - Input: payloads.MustEncodeSingle(op.request), - }, - }, - }) + commands = append(commands, nexusCommand(op)) } c.respond(commands...) var ids []int64 @@ -115,31 +114,67 @@ func (c *streamCaller) schedule(ops ...streamOp) []int64 { return ids } +// completeTask completes the open Workflow Task with commands. It polls no next task, since the +// commands may end the run. +func (c *streamCaller) completeTask(commands ...*commandpb.Command) error { + _, err := c.env.FrontendClient().RespondWorkflowTaskCompleted(c.s.Context(), + &workflowservice.RespondWorkflowTaskCompletedRequest{ + Identity: "test", + TaskToken: c.token, + Commands: commands, + }) + return err +} + func (c *streamCaller) respond(commands ...*commandpb.Command) { - _, err := c.env.FrontendClient().RespondWorkflowTaskCompleted(c.s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Identity: "test", - TaskToken: c.token, - Commands: commands, - }) - c.s.NoError(err) + c.s.NoError(c.completeTask(commands...)) c.poll() } +// continueAsNew continues the run as new on the caller's task queue. +func (c *streamCaller) continueAsNew() *commandpb.Command { + taskQueue := &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + attributes := &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, + TaskQueue: taskQueue, + } + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ + ContinueAsNewWorkflowExecutionCommandAttributes: attributes, + }, + } +} + +// completeCommand completes the run with result. +func completeCommand(result *commonpb.Payloads) *commandpb.Command { + attributes := &commandpb.CompleteWorkflowExecutionCommandAttributes{Result: result} + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: attributes, + }, + } +} + // outcome waits for the end of the operation that scheduledID scheduled. It answers whether the // operation started async first. func (c *streamCaller) outcome(scheduledID int64, result proto.Message) (*failurepb.Failure, bool) { for { started := false for _, event := range c.history { - if attrs := event.GetNexusOperationStartedEventAttributes(); attrs != nil && attrs.GetScheduledEventId() == scheduledID { + if attrs := event.GetNexusOperationStartedEventAttributes(); attrs != nil && + attrs.GetScheduledEventId() == scheduledID { started = true } - if attrs := event.GetNexusOperationCompletedEventAttributes(); attrs != nil && attrs.GetScheduledEventId() == scheduledID { + if attrs := event.GetNexusOperationCompletedEventAttributes(); attrs != nil && + attrs.GetScheduledEventId() == scheduledID { c.s.NoError(sdkconverter.PreferProtoDataConverter.FromPayloads( &commonpb.Payloads{Payloads: []*commonpb.Payload{attrs.GetResult()}}, result)) return nil, started } - if attrs := event.GetNexusOperationFailedEventAttributes(); attrs != nil && attrs.GetScheduledEventId() == scheduledID { + if attrs := event.GetNexusOperationFailedEventAttributes(); attrs != nil && + attrs.GetScheduledEventId() == scheduledID { return attrs.GetFailure(), started } } @@ -157,6 +192,22 @@ type streamOp struct { request proto.Message } +// nexusCommand schedules op as a System Nexus operation. +func nexusCommand(op streamOp) *commandpb.Command { + attributes := &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: op.name, + Input: payloads.MustEncodeSingle(op.request), + } + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: attributes, + }, + } +} + func ref(id string) *streamservice.StreamRef { return &streamservice.StreamRef{Id: id} } @@ -164,7 +215,10 @@ func ref(id string) *streamservice.StreamRef { func records(topics ...string) []*streamservice.AppendRecord { out := make([]*streamservice.AppendRecord, 0, len(topics)) for _, topic := range topics { - out = append(out, &streamservice.AppendRecord{Topic: topic, Body: payload.EncodeString(topic)}) + out = append( + out, + &streamservice.AppendRecord{Topic: topic, Body: payload.EncodeString(topic)}, + ) } return out } @@ -213,44 +267,79 @@ func (s *StreamTestSuite) TestWorkflowReadAndDescribe() { id := testcore.RandomizeStr("chat") writer := workflowWriter("run-1", 3) - _, err := sc.CreateStream(ctx, &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id, Owner: writer}) + _, err := sc.CreateStream(ctx, &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) s.NoError(err) appended, err := sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: writer, Sequence: 1, Records: records("turns", "tools"), + Namespace: ns, Stream: ref(id), Producer: writer, Sequence: 1, + Records: records("turns", "tools"), }) s.NoError(err) var page streamservice.ReadRecordsResponse - failure, started := c.outcome(c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id)}})[0], &page) + failure, started := c.outcome( + c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id)}})[0], + &page, + ) s.Nil(failure) s.False(started) s.Equal([]string{"turns", "tools"}, kindsAndTopics(&page)) var described streamservice.DescribeStreamResponse - s.Nil(c.call(streamOp{"DescribeStream", &streamservice.DescribeStreamRequest{Stream: ref(id)}}, &described)) + s.Nil( + c.call( + streamOp{"DescribeStream", &streamservice.DescribeStreamRequest{Stream: ref(id)}}, + &described, + ), + ) s.Equal(appended.GetLastOffset(), described.GetLastOffset()) - _, err = sc.CloseStream(ctx, &streamservice.CloseStreamRequest{Namespace: ns, Stream: ref(id), Producer: writer, Result: payload.EncodeString("done")}) + _, err = sc.CloseStream( + ctx, + &streamservice.CloseStreamRequest{ + Namespace: ns, + Stream: ref(id), + Producer: clientProducer(), + Result: payload.EncodeString("done"), + }, + ) s.NoError(err) var last streamservice.ReadRecordsResponse - s.Nil(c.call(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id), FromOffset: page.GetNextOffset()}}, &last)) + s.Nil( + c.call( + streamOp{ + "ReadRecords", + &streamservice.ReadRecordsRequest{ + Stream: ref(id), + FromOffset: page.GetNextOffset(), + }, + }, + &last, + ), + ) s.Equal([]string{"close"}, kindsAndTopics(&last)) s.True(last.GetDone()) } -func (s *StreamTestSuite) TestWorkflowReadWaitsForAnAppend() { +// A Workflow's wake parks until a Record passes its filter, and answers with no Records. +func (s *StreamTestSuite) TestWorkflowWakeWaitsForAnAppend() { env := s.newEnv() sc := s.streamClient(env) c := s.newCaller(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) - read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id), Topics: []string{"turns"}}})[0] + read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: ref(id), Topics: []string{"turns"}, WakeOnly: true, + }})[0] for i, topic := range []string{"other", "turns"} { _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: int64(i + 1), Records: records(topic), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: int64(i + 1), + Records: records(topic), }) s.NoError(err) } @@ -259,40 +348,61 @@ func (s *StreamTestSuite) TestWorkflowReadWaitsForAnAppend() { failure, started := c.outcome(read, &page) s.Nil(failure) s.True(started) - s.Equal([]string{"turns"}, kindsAndTopics(&page)) + s.Empty(page.GetRecords()) + s.False(page.GetDone()) } -func (s *StreamTestSuite) TestWorkflowReadEndsOnClose() { +func (s *StreamTestSuite) TestWorkflowWakeEndsOnClose() { env := s.newEnv() sc := s.streamClient(env) c := s.newCaller(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) - read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id)}})[0] - _, err = sc.CloseStream(s.Context(), &streamservice.CloseStreamRequest{Namespace: ns, Stream: ref(id), Producer: clientProducer()}) + read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: ref(id), WakeOnly: true, + }})[0] + _, err = sc.CloseStream( + s.Context(), + &streamservice.CloseStreamRequest{ + Namespace: ns, + Stream: ref(id), + Producer: clientProducer(), + }, + ) s.NoError(err) var page streamservice.ReadRecordsResponse failure, started := c.outcome(read, &page) s.Nil(failure) s.True(started) - s.Equal([]string{"close"}, kindsAndTopics(&page)) + s.Empty(page.GetRecords()) s.True(page.GetDone()) } func (s *StreamTestSuite) TestWorkflowReadOfAMissingStreamFails() { env := s.newEnv() c := s.newCaller(env) - failure := c.call(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(testcore.RandomizeStr("missing"))}}, - &streamservice.ReadRecordsResponse{}) + failure := c.call( + streamOp{ + "ReadRecords", + &streamservice.ReadRecordsRequest{Stream: ref(testcore.RandomizeStr("missing"))}, + }, + &streamservice.ReadRecordsResponse{}, + ) s.Equal(stream.ReasonNotFound, causeType(failure)) } func (s *StreamTestSuite) streamClient(env *testcore.TestEnv) streamservice.StreamServiceClient { - conn, err := grpc.NewClient(env.FrontendGRPCAddress(), grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient( + env.FrontendGRPCAddress(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) s.NoError(err) s.T().Cleanup(func() { _ = conn.Close() }) return streamservice.NewStreamServiceClient(conn) @@ -313,7 +423,10 @@ func (s *StreamTestSuite) TestClientCreateAppendReadClose() { ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - created, err := sc.CreateStream(ctx, &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + created, err := sc.CreateStream( + ctx, + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) s.True(created.GetCreated()) generated, err := sc.CreateStream(ctx, &streamservice.CreateStreamRequest{Namespace: ns}) @@ -321,34 +434,61 @@ func (s *StreamTestSuite) TestClientCreateAppendReadClose() { s.NotEmpty(generated.GetStream().GetId()) appended, err := sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, Records: records("turns", "tool/search"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("turns", "tool/search"), }) s.NoError(err) resent, err := sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, Records: records("turns", "tool/search"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("turns", "tool/search"), }) s.NoError(err) s.Equal(appended.GetFirstOffset(), resent.GetFirstOffset()) - page, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id), TopicPrefixes: []string{"tool/"}}) + page, err := sc.ReadRecords( + ctx, + &streamservice.ReadRecordsRequest{ + Namespace: ns, + Stream: ref(id), + TopicPrefixes: []string{"tool/"}, + }, + ) s.NoError(err) s.Equal([]string{"tool/search"}, kindsAndTopics(page)) s.Equal(appended.GetLastOffset()+1, page.GetNextOffset()) - described, err := sc.DescribeStream(ctx, &streamservice.DescribeStreamRequest{Namespace: ns, Stream: ref(id)}) + described, err := sc.DescribeStream( + ctx, + &streamservice.DescribeStreamRequest{Namespace: ns, Stream: ref(id)}, + ) s.NoError(err) s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) s.Equal(appended.GetLastOffset(), described.GetLastOffset()) - _, err = sc.CloseStream(ctx, &streamservice.CloseStreamRequest{Namespace: ns, Stream: ref(id), Producer: clientProducer()}) + _, err = sc.CloseStream( + ctx, + &streamservice.CloseStreamRequest{ + Namespace: ns, + Stream: ref(id), + Producer: clientProducer(), + }, + ) s.NoError(err) - last, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id), FromOffset: page.GetNextOffset()}) + last, err := sc.ReadRecords( + ctx, + &streamservice.ReadRecordsRequest{ + Namespace: ns, + Stream: ref(id), + FromOffset: page.GetNextOffset(), + }, + ) s.NoError(err) s.Equal([]string{"close"}, kindsAndTopics(last)) s.True(last.GetDone()) _, err = sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 2, Records: records("late"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 2, + Records: records("late"), }) st := status.Convert(err) s.Equal(codes.FailedPrecondition, st.Code()) @@ -372,7 +512,8 @@ func (s *StreamTestSuite) TestClientReadWaitsForAnAppend() { done := make(chan result, 1) go func() { resp, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{ - Namespace: ns, Stream: ref(id), Start: streamservice.READ_START_END, Wait: durationpb.New(30 * time.Second), + Namespace: ns, Stream: ref(id), Start: streamservice.READ_START_END, + Wait: durationpb.New(30 * time.Second), }) done <- result{resp, err} }() @@ -381,7 +522,8 @@ func (s *StreamTestSuite) TestClientReadWaitsForAnAppend() { s.Await(func(s *StreamTestSuite) { sequence++ _, err := sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: sequence, Records: records("turns"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: sequence, + Records: records("turns"), }) s.NoError(err) select { @@ -416,7 +558,7 @@ func (s *StreamTestSuite) TestClientReadWaitsForTheCreate() { s.NoError(<-done) } -func (s *StreamTestSuite) TestWorkflowReadWaitsForTheCreate() { +func (s *StreamTestSuite) TestWorkflowWakeWaitsForTheCreate() { env := s.newEnv() sc := s.streamClient(env) c := s.newCaller(env) @@ -425,12 +567,17 @@ func (s *StreamTestSuite) TestWorkflowReadWaitsForTheCreate() { read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ Stream: ref(id), + WakeOnly: true, CreateWait: durationpb.New(30 * time.Second), }})[0] - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, Records: records("turns"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("turns"), }) s.NoError(err) @@ -438,24 +585,76 @@ func (s *StreamTestSuite) TestWorkflowReadWaitsForTheCreate() { failure, started := c.outcome(read, &page) s.Nil(failure) s.True(started) - s.Equal([]string{"turns"}, kindsAndTopics(&page)) + s.Empty(page.GetRecords()) +} + +// A wake whose create wait ends first leaves the Workflow Stream's node abandoned. A later wake +// parks again and the create opens it, since the node has no execution of its own to replace. +func (s *StreamTestSuite) TestAbandonedWorkflowStreamCanStillBeCreated() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + ns := env.Namespace().String() + id := testcore.RandomizeStr("chat") + owned := &streamservice.StreamRef{ + Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID(), + } + + failure := c.call(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, + CreateWait: durationpb.New(time.Second), Wait: durationpb.New(time.Hour), + }}, &streamservice.ReadRecordsResponse{}) + s.Equal(stream.ReasonNotFound, causeType(failure)) + + wake := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, CreateWait: durationpb.New(30 * time.Second), + }})[0] + _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ + Namespace: ns, StreamId: id, Owner: workflowWriter(c.current, 3), Ref: owned, + CloseWithRun: true, + }) + s.NoError(err) + _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ + Namespace: ns, Stream: owned, Producer: clientProducer(), Sequence: 1, + Records: records("turns"), + }) + s.NoError(err) + failure, started := c.outcome(wake, &streamservice.ReadRecordsResponse{}) + s.Nil(failure) + s.True(started) } // closeWithRun creates a Stream that closes with the caller's run chain. -func (s *StreamTestSuite) closeWithRun(sc streamservice.StreamServiceClient, env *testcore.TestEnv, c *streamCaller) *streamservice.StreamRef { +func (s *StreamTestSuite) closeWithRun( + sc streamservice.StreamServiceClient, + env *testcore.TestEnv, + c *streamCaller, +) *streamservice.StreamRef { id := testcore.RandomizeStr("chat") _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ - Namespace: env.Namespace().String(), - StreamId: id, - Owner: workflowWriter(c.run.GetRunID(), 3), - Ref: &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID()}, + Namespace: env.Namespace().String(), + StreamId: id, + Owner: workflowWriter(c.run.GetRunID(), 3), + Ref: &streamservice.StreamRef{ + Id: id, + WorkflowId: c.run.GetID(), + FirstExecutionRunId: c.run.GetRunID(), + }, CloseWithRun: true, }) s.NoError(err) - return &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID()} + return &streamservice.StreamRef{ + Id: id, + WorkflowId: c.run.GetID(), + FirstExecutionRunId: c.run.GetRunID(), + } } -func (s *StreamTestSuite) readClose(sc streamservice.StreamServiceClient, env *testcore.TestEnv, owned *streamservice.StreamRef) *streamservice.StreamRecord { +func (s *StreamTestSuite) readClose( + sc streamservice.StreamServiceClient, + env *testcore.TestEnv, + owned *streamservice.StreamRef, +) *streamservice.StreamRecord { page, err := sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{ Namespace: env.Namespace().String(), Stream: owned, Wait: durationpb.New(20 * time.Second), }) @@ -470,19 +669,7 @@ func (s *StreamTestSuite) TestStreamClosesWithTheOwnerResult() { c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) - _, err := env.FrontendClient().RespondWorkflowTaskCompleted(s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Identity: "test", - TaskToken: c.token, - Commands: []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{ - Result: payloads.EncodeString("answer"), - }, - }, - }}, - }) - s.NoError(err) + s.NoError(c.completeTask(completeCommand(payloads.EncodeString("answer")))) closed := s.readClose(sc, env, owned) s.Equal(payload.EncodeString("answer").GetData(), closed.GetCloseResult().GetData()) @@ -495,15 +682,116 @@ func (s *StreamTestSuite) TestStreamClosesWithATerminatedOwner() { c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) - s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), c.run.GetRunID(), "stop")) + s.NoError( + env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), c.run.GetRunID(), "stop"), + ) closed := s.readClose(sc, env, owned) s.NotNil(closed.GetCloseFailure().GetTerminatedFailureInfo()) s.Equal(streamservice.CLOSE_STATUS_TERMINATED, closed.GetCloseStatus()) } -// A cron run's end isn't the chain's end, so the next cron run writes on. The terminate of a later -// run ends the chain and closes the Stream with its status. +// closeOp closes a Workflow Stream through System Nexus, as the calling run. +func closeOp(owned *streamservice.StreamRef, result string) streamOp { + return streamOp{ + "CloseStream", + &streamservice.CloseStreamRequest{Stream: owned, Result: payload.EncodeString(result)}, + } +} + +func (s *StreamTestSuite) TestWorkflowCloseCommandClosesItsStream() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + owned := s.closeWithRun(sc, env, c) + + s.Nil(c.call(closeOp(owned, "answer"), &streamservice.CloseStreamResponse{})) + + closed := s.readClose(sc, env, owned) + s.Equal(payload.EncodeString("answer").GetData(), closed.GetCloseResult().GetData()) + s.Equal(streamservice.CLOSE_STATUS_COMPLETED, closed.GetCloseStatus()) + s.Equal("wf:"+stream.EncodeIDPart(c.run.GetRunID()), closed.GetProducer().GetId()) +} + +func (s *StreamTestSuite) TestCloseCommandOfAnotherChainIsRefused() { + env := s.newEnv() + sc := s.streamClient(env) + owned := s.closeWithRun(sc, env, s.newCaller(env)) + + failure := s.newCaller(env).call(closeOp(owned, "answer"), &streamservice.CloseStreamResponse{}) + s.Equal(stream.ReasonNotOwner, causeType(failure)) + + described, err := sc.DescribeStream( + s.Context(), + &streamservice.DescribeStreamRequest{Namespace: env.Namespace().String(), Stream: owned}, + ) + s.NoError(err) + s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) +} + +// A direct System Nexus call waits for no Workflow Task to commit, so it can't close the Stream, +// even with the link of the owner run. +func (s *StreamTestSuite) TestDirectCloseCallIsRefused() { + env := s.newEnv() + sc := s.streamClient(env) + ns := env.Namespace().String() + c := s.newCaller(env) + owned := s.closeWithRun(sc, env, c) + + resp, err := env.FrontendClient(). + StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ + Namespace: ns, + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: "CloseStream", + Input: payloads.MustEncodeSingle(closeOp(owned, "answer").request), + Links: []*commonpb.Link{readerLink(env, c.run)}, + }) + s.NoError(err) + s.Equal(stream.ReasonInvalid, causeType(resp.GetOperationFailure().GetFailure())) + + described, err := sc.DescribeStream( + s.Context(), + &streamservice.DescribeStreamRequest{Namespace: ns, Stream: owned}, + ) + s.NoError(err) + s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) +} + +// A bad command fails the Workflow Task, so the server drops the close command that came with it. +// The retried task's close command closes the Stream once. +func (s *StreamTestSuite) TestCloseCommandOfAFailedWorkflowTaskIsDropped() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + owned := s.closeWithRun(sc, env, c) + + // An Activity with empty attributes is a bad command. + empty := &commandpb.ScheduleActivityTaskCommandAttributes{} + bad := &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: empty, + }, + } + s.Error(c.completeTask(nexusCommand(closeOp(owned, "failed task")), bad)) + c.poll() + + s.Nil(c.call(closeOp(owned, "retried task"), &streamservice.CloseStreamResponse{})) + page, err := sc.ReadRecords( + s.Context(), + &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: owned}, + ) + s.NoError(err) + s.Equal([]string{"close"}, kindsAndTopics(page)) + s.Equal( + payload.EncodeString("retried task").GetData(), + page.GetRecords()[0].GetCloseResult().GetData(), + ) +} + +// A cron run's end isn't the chain's end, so the next cron run still writes. The terminate of a +// later run ends the chain and closes the Stream with its status. func (s *StreamTestSuite) TestCronRunEndKeepsTheStreamOpen() { env := s.newEnv() sc := s.streamClient(env) @@ -520,13 +808,7 @@ func (s *StreamTestSuite) TestCronRunEndKeepsTheStreamOpen() { owned := s.closeWithRun(sc, env, c) // Run 1 completes and the next cron run starts. - c.respond(&commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb. - CompleteWorkflowExecutionCommandAttributes{}, - }, - }) + c.respond(completeCommand(nil)) appended, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ Namespace: env.Namespace().String(), Stream: owned, Producer: workflowWriter(c.current, 3), Sequence: 1, Records: records("next"), @@ -561,43 +843,92 @@ func (s *StreamTestSuite) TestDeferredWorkflowBatchIsWithdrawnByAFailedTask() { }) s.NoError(err) _, err = sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: workflowWriter("run-1", 7), Sequence: 1, Records: records("b"), + Namespace: ns, Stream: ref(id), Producer: workflowWriter("run-1", 7), Sequence: 1, + Records: records("b"), PreviousCompletedTask: &streamservice.WorkflowTaskEpoch{StartedEventId: 2, Attempt: 1}, FailedTasks: []int64{3}, }) s.NoError(err) - page, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id)}) + page, err := sc.ReadRecords( + ctx, + &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id)}, + ) s.NoError(err) s.Equal([]string{"a", "", "b"}, kindsAndTopics(page)) s.Equal(streamservice.STREAM_RECORD_KIND_SUPERSEDED, page.GetRecords()[1].GetKind()) s.Equal([]int64{3}, page.GetRecords()[1].GetSupersession().GetFailedTasks()) } -func (s *StreamTestSuite) TestWorkflowReadEndsAtItsWait() { +// A Workflow read that isn't a wake never parks. It answers what the Stream has at once, an +// empty page here, so no parked read keeps a page. +func (s *StreamTestSuite) TestWorkflowReadThatIsNotAWakeAnswersAtOnce() { env := s.newEnv() sc := s.streamClient(env) c := s.newCaller(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) appended, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, Records: records("other"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("other"), }) s.NoError(err) var page streamservice.ReadRecordsResponse - failure, started := c.outcome(c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ - Stream: ref(id), Topics: []string{"turns"}, Wait: durationpb.New(time.Second), - }})[0], &page) + failure, started := c.outcome( + c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: ref(id), Topics: []string{"turns"}, Wait: durationpb.New(time.Minute), + }})[0], + &page, + ) s.Nil(failure) - s.True(started) + s.False(started) s.Empty(page.GetRecords()) s.False(page.GetDone()) s.Equal(appended.GetLastOffset()+1, page.GetNextOffset()) } +// One large append must not copy its batch into every waiting read, which would push the +// execution past the mutable state limit and expire the Stream. +func (s *StreamTestSuite) TestLargeAppendWithManyReadersStaysUnderTheStateLimit() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + ns := env.Namespace().String() + id := testcore.RandomizeStr("chat") + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) + s.NoError(err) + + ops := make([]streamOp, 0, 5) + for range 5 { + ops = append(ops, streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: ref(id), Start: streamservice.READ_START_END, Wait: durationpb.New(time.Minute), + }}) + } + c.schedule(ops...) + + big := make([]byte, 1800*1024) + _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: []*streamservice.AppendRecord{{Topic: "big", Body: &commonpb.Payload{Data: big}}}, + }) + s.NoError(err) + // The size check runs in the next transition, such as a wake's callback task. + time.Sleep(2 * time.Second) //nolint:forbidigo // The callback tasks run on real time. + described, err := sc.DescribeStream(s.Context(), + &streamservice.DescribeStreamRequest{Namespace: ns, Stream: ref(id)}) + s.NoError(err) + s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) +} + // An append to a Workflow Stream completes the Workflow's parked read in the Workflow's // transaction, since the Stream nests in the Workflow. func (s *StreamTestSuite) TestNestedStreamWakesItsWorkflow() { @@ -606,7 +937,9 @@ func (s *StreamTestSuite) TestNestedStreamWakesItsWorkflow() { c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) - read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: owned}})[0] + read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, + }})[0] _, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ Namespace: env.Namespace().String(), Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), Sequence: 1, Records: records("turns"), @@ -617,10 +950,16 @@ func (s *StreamTestSuite) TestNestedStreamWakesItsWorkflow() { failure, started := c.outcome(read, &page) s.Nil(failure) s.True(started) - s.Equal([]string{"turns"}, kindsAndTopics(&page)) + s.Empty(page.GetRecords()) // The Stream id names no execution of its own. - _, err = sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{Namespace: env.Namespace().String(), Stream: ref(owned.GetId())}) + _, err = sc.DescribeStream( + s.Context(), + &streamservice.DescribeStreamRequest{ + Namespace: env.Namespace().String(), + Stream: ref(owned.GetId()), + }, + ) s.Equal(codes.NotFound, status.Code(err)) } @@ -631,15 +970,7 @@ func (s *StreamTestSuite) TestStreamClosesWithTheLastRunOfTheChain() { owned := s.closeWithRun(sc, env, c) // Run 1 continues as new. The Stream stays open in run 1's tree. - c.respond(&commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ - ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, - TaskQueue: &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - }, - }, - }) + c.respond(c.continueAsNew()) described, err := sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{ Namespace: env.Namespace().String(), Stream: owned, }) @@ -647,24 +978,79 @@ func (s *StreamTestSuite) TestStreamClosesWithTheLastRunOfTheChain() { s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) // Run 2 completes, which ends the chain. - _, err = env.FrontendClient().RespondWorkflowTaskCompleted(s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Identity: "test", - TaskToken: c.token, - Commands: []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{ - Result: payloads.EncodeString("run-2"), - }, - }, - }}, - }) - s.NoError(err) + s.NoError(c.completeTask(completeCommand(payloads.EncodeString("run-2")))) closed := s.readClose(sc, env, owned) s.Equal(payload.EncodeString("run-2").GetData(), closed.GetCloseResult().GetData()) s.Equal(streamservice.CLOSE_STATUS_COMPLETED, closed.GetCloseStatus()) } +// A reset to an earlier run of a continued-as-new chain terminates the current run, but the +// chain continues in the reset run, so its Streams stay open. +func (s *StreamTestSuite) TestResetToAnEarlierRunKeepsTheStreamOpen() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + ns := env.Namespace().String() + owned := s.closeWithRun(sc, env, c) + first := c.run.GetRunID() + c.respond(c.continueAsNew()) + s.NotEqual(first, c.current) + + reset, err := env.FrontendClient().ResetWorkflowExecution(s.Context(), + &workflowservice.ResetWorkflowExecutionRequest{ + Namespace: ns, + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: c.run.GetID(), + RunId: first, + }, + Reason: "test", + WorkflowTaskFinishEventId: 4, + RequestId: uuid.NewString(), + }) + s.NoError(err) + s.T().Cleanup(func() { + _ = env.SdkClient(). + TerminateWorkflow(s.Context(), c.run.GetID(), reset.GetRunId(), "test cleanup") + }) + + // The chain's end would close the Stream through a task on the terminated run. + time.Sleep(2 * time.Second) //nolint:forbidigo // The close task runs on real time. + described, err := sc.DescribeStream(s.Context(), + &streamservice.DescribeStreamRequest{Namespace: ns, Stream: owned}) + s.NoError(err) + s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) + + // The reset run's end is the chain's end. + s.NoError( + env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), reset.GetRunId(), "stop"), + ) + s.Equal(streamservice.CLOSE_STATUS_TERMINATED, s.readClose(sc, env, owned).GetCloseStatus()) +} + +// A chain whose first run has no Stream ends with no write to that run. +func (s *StreamTestSuite) TestChainEndLeavesAFirstRunWithNoStreamAlone() { + env := s.newEnv() + c := s.newCaller(env) + ns := env.Namespace().String() + first := c.run.GetRunID() + c.respond(c.continueAsNew()) + transitions := func() int64 { + resp, err := env.AdminClient().DescribeMutableState(s.Context(), + &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: c.run.GetID(), RunId: first}, + Archetype: chasm.WorkflowArchetype, + }) + s.NoError(err) + return resp.GetDatabaseMutableState().GetExecutionInfo().GetStateTransitionCount() + } + before := transitions() + + s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), c.current, "stop")) + time.Sleep(2 * time.Second) //nolint:forbidigo // The chain's close task runs on real time. + s.Equal(before, transitions()) +} + // directRead reads through the direct System Nexus call, which adds no History event. // A link names the reading Workflow run. func (s *StreamTestSuite) directRead( @@ -672,29 +1058,38 @@ func (s *StreamTestSuite) directRead( req *streamservice.ReadRecordsRequest, links ...*commonpb.Link, ) (*streamservice.ReadRecordsResponse, *failurepb.Failure) { - resp, err := env.FrontendClient().StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ - Namespace: env.Namespace().String(), - Endpoint: commonnexus.SystemEndpoint, - Service: stream.ServiceName, - Operation: "ReadRecords", - Input: payloads.MustEncodeSingle(req), - Links: links, - }) + resp, err := env.FrontendClient(). + StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ + Namespace: env.Namespace().String(), + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: "ReadRecords", + Input: payloads.MustEncodeSingle(req), + Links: links, + }) s.NoError(err) if failure := resp.GetOperationFailure().GetFailure(); failure != nil { return nil, failure } var page streamservice.ReadRecordsResponse s.NoError(sdkconverter.PreferProtoDataConverter.FromPayloads( - &commonpb.Payloads{Payloads: []*commonpb.Payload{resp.GetSyncSuccess().GetPayload()}}, &page)) + &commonpb.Payloads{ + Payloads: []*commonpb.Payload{resp.GetSyncSuccess().GetPayload()}, + }, + &page, + )) return &page, nil } func (s *StreamTestSuite) historyLength(env *testcore.TestEnv, c *streamCaller) int { - resp, err := env.FrontendClient().GetWorkflowExecutionHistory(s.Context(), &workflowservice.GetWorkflowExecutionHistoryRequest{ - Namespace: env.Namespace().String(), - Execution: &commonpb.WorkflowExecution{WorkflowId: c.run.GetID(), RunId: c.run.GetRunID()}, - }) + resp, err := env.FrontendClient().GetWorkflowExecutionHistory(s.Context(), + &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{ + WorkflowId: c.run.GetID(), + RunId: c.run.GetRunID(), + }, + }) s.NoError(err) return len(resp.GetHistory().GetEvents()) } @@ -710,12 +1105,15 @@ func (s *StreamTestSuite) TestWorkflowReadsDirectlyWithNoRecordsInHistory() { ns := env.Namespace().String() appendTopics := func(sequence int64, topics ...string) { _, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), Sequence: sequence, Records: records(topics...), + Namespace: ns, Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), + Sequence: sequence, Records: records(topics...), }) s.NoError(err) } - wake := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: owned, WakeOnly: true}})[0] + wake := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, + }})[0] appendTopics(1, "a", "b") var woke streamservice.ReadRecordsResponse failure, started := c.outcome(wake, &woke) @@ -725,31 +1123,65 @@ func (s *StreamTestSuite) TestWorkflowReadsDirectlyWithNoRecordsInHistory() { s.Equal(int64(0), woke.GetNextOffset()) before := s.historyLength(env, c) - page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: woke.GetNextOffset()}) + page, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: woke.GetNextOffset()}, + ) s.Nil(failure) s.Equal([]string{"a", "b"}, kindsAndTopics(page)) s.Equal(before, s.historyLength(env, c)) appendTopics(2, "c") - again, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, ToOffset: page.GetNextOffset()}) + again, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, ToOffset: page.GetNextOffset()}, + ) s.Nil(failure) s.Equal(page.GetRecords(), again.GetRecords()) s.Equal(page.GetNextOffset(), again.GetNextOffset()) // A direct read with wait long-polls, as the frontend read does. waited, failure := s.directRead(env, &streamservice.ReadRecordsRequest{ - Stream: owned, Start: streamservice.READ_START_END, Wait: durationpb.New(500 * time.Millisecond), + Stream: owned, Start: streamservice.READ_START_END, + Wait: durationpb.New(500 * time.Millisecond), }) s.Nil(failure) s.Empty(waited.GetRecords()) - _, failure = s.directRead(env, &streamservice.ReadRecordsRequest{Stream: ref(testcore.RandomizeStr("missing"))}) + _, failure = s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: ref(testcore.RandomizeStr("missing"))}, + ) s.Equal(stream.ReasonNotFound, causeType(failure)) } // The first run of a chain keeps the chain's Streams after it closes, so the namespace retention // can't delete them while a later run still writes. func (s *StreamTestSuite) TestChainOutlivesTheRetentionOfItsFirstRun() { + env := s.retentionEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + owned := s.closeWithRun(sc, env, c) + c.respond(c.continueAsNew()) + // Several retentions of the first run pass while run 2 is open. + <-time.After(4 * time.Second) + + _, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ + Namespace: env.Namespace().String(), Stream: owned, + Producer: workflowWriter(c.current, 3), Sequence: 1, Records: records("late"), + }) + s.NoError(err) + page, err := sc.ReadRecords( + s.Context(), + &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: owned}, + ) + s.NoError(err) + s.Equal([]string{"late"}, kindsAndTopics(page)) +} + +// retentionEnv is a cluster whose namespace retention is 1 s, so a retention delete shows in +// seconds. +func (s *StreamTestSuite) retentionEnv() *testcore.TestEnv { env := testcore.NewEnv(s.T(), testcore.WithDedicatedCluster(), testcore.WithDynamicConfig(stream.Enabled, true), @@ -757,37 +1189,201 @@ func (s *StreamTestSuite) TestChainOutlivesTheRetentionOfItsFirstRun() { testcore.WithDynamicConfig(dynamicconfig.NamespaceMinRetentionLocal, time.Duration(0)), testcore.WithDynamicConfig(dynamicconfig.RetentionTimerJitterDuration, time.Duration(0)), ) - _, err := env.FrontendClient().UpdateNamespace(s.Context(), &workflowservice.UpdateNamespaceRequest{ - Namespace: env.Namespace().String(), - Config: &namespacepb.NamespaceConfig{WorkflowExecutionRetentionTtl: durationpb.New(time.Second)}, - }) + _, err := env.FrontendClient().UpdateNamespace(s.Context(), + &workflowservice.UpdateNamespaceRequest{ + Namespace: env.Namespace().String(), + Config: &namespacepb.NamespaceConfig{ + WorkflowExecutionRetentionTtl: durationpb.New(time.Second), + }, + }) s.NoError(err) // Wait for the namespace cache to see the new retention. <-time.After(2 * testcore.NamespaceCacheRefreshInterval) + return env +} - sc := s.streamClient(env) - c := s.newCaller(env) - owned := s.closeWithRun(sc, env, c) - c.respond(&commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ - ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, - TaskQueue: &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - }, +// activityStream is a Stream of a Standalone Activity's first attempt, with one Record. +type activityStream struct { + ref *streamservice.StreamRef + task *workflowservice.PollActivityTaskQueueResponse +} + +func (s *StreamTestSuite) startActivityStream( + env *testcore.TestEnv, + retention *durationpb.Duration, +) activityStream { + ctx := s.Context() + ns := env.Namespace().String() + activityID := "tool-" + uuid.NewString() + taskQueue := &taskqueuepb.TaskQueue{ + Name: "tools-" + uuid.NewString(), + Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + } + started, err := env.FrontendClient().StartActivityExecution(ctx, + &workflowservice.StartActivityExecutionRequest{ + Namespace: ns, + ActivityId: activityID, + ActivityType: &commonpb.ActivityType{Name: "tool"}, + Identity: "test", + TaskQueue: taskQueue, + StartToCloseTimeout: durationpb.New(time.Minute), + }) + s.NoError(err) + task, err := env.FrontendClient().PollActivityTaskQueue(ctx, + &workflowservice.PollActivityTaskQueueRequest{ + Namespace: ns, TaskQueue: taskQueue, Identity: "test", + }) + s.NoError(err) + id := testcore.RandomizeStr("tokens") + owned := &streamservice.StreamRef{ + Id: id, ActivityId: activityID, ActivityRunId: started.GetRunId(), + } + owner := &streamservice.Producer{ + Kind: streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY, + Id: "sa:" + activityID + ":" + started.GetRunId(), + Epoch: &streamservice.ProducerEpoch{ + AttemptScheduledTime: task.GetCurrentAttemptScheduledTime(), + Attempt: task.GetAttempt(), }, + } + sc := s.streamClient(env) + _, err = sc.CreateStream(ctx, &streamservice.CreateStreamRequest{ + Namespace: ns, StreamId: id, Ref: owned, Owner: owner, Retention: retention, + TaskToken: task.GetTaskToken(), }) - // Several retentions of the first run pass while run 2 is open. + s.NoError(err) + _, err = sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ + Namespace: ns, Stream: owned, Producer: owner, Sequence: 1, Records: records("tokens"), + TaskToken: task.GetTaskToken(), + }) + s.NoError(err) + return activityStream{ref: owned, task: task} +} + +// completeActivity completes the attempt of an Activity Stream, which closes the Stream. +func (s *StreamTestSuite) completeActivity(env *testcore.TestEnv, a activityStream) { + _, err := env.FrontendClient().RespondActivityTaskCompleted(s.Context(), + &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), TaskToken: a.task.GetTaskToken(), Identity: "test", + }) + s.NoError(err) +} + +// A closed Standalone Activity keeps its Streams readable for their Retention, so the namespace +// retention can't delete them with the Activity. +func (s *StreamTestSuite) TestActivityStreamOutlivesTheRetentionOfItsActivity() { + env := s.retentionEnv() + a := s.startActivityStream(env, nil) + s.completeActivity(env, a) + // Several namespace retentions pass after the Activity's end. <-time.After(4 * time.Second) - _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: env.Namespace().String(), Stream: owned, - Producer: workflowWriter(c.current, 3), Sequence: 1, Records: records("late"), - }) + page, err := s.streamClient(env).ReadRecords(s.Context(), + &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: a.ref}) s.NoError(err) - page, err := sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: owned}) + s.Equal([]string{"tokens", "close"}, kindsAndTopics(page)) +} + +// A closed Activity's Stream expires on its own timer, and then the namespace retention deletes +// the Activity, though nobody reads the Stream. +func (s *StreamTestSuite) TestClosedActivityIsDeletedAfterItsStreamExpires() { + env := s.retentionEnv() + a := s.startActivityStream(env, durationpb.New(time.Second)) + s.completeActivity(env, a) + + s.AwaitTrue(func() bool { + _, err := env.FrontendClient().DescribeActivityExecution(s.Context(), + &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: a.ref.GetActivityId(), + RunId: a.ref.GetActivityRunId(), + }) + var notFound *serviceerror.NotFound + return errors.As(err, ¬Found) + }, 20*time.Second, 200*time.Millisecond) +} + +// A chain's first run keeps its Streams after it continues as new. Their tasks still run there, +// so an unread closed Stream expires, and then the namespace retention deletes the first run. +func (s *StreamTestSuite) TestUnreadFirstRunIsDeletedAfterItsStreamExpires() { + env := s.retentionEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + ns := env.Namespace().String() + id := testcore.RandomizeStr("chat") + first := c.run.GetRunID() + owned := &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: first} + _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ + Namespace: ns, StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), + Owner: workflowWriter(first, 3), CloseWithRun: true, + }) s.NoError(err) - s.Equal([]string{"late"}, kindsAndTopics(page)) + c.respond(c.continueAsNew()) + s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), c.current, "stop")) + + s.AwaitTrue(func() bool { + _, err := env.AdminClient().DescribeMutableState(s.Context(), + &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: c.run.GetID(), RunId: first}, + Archetype: chasm.WorkflowArchetype, + }) + var notFound *serviceerror.NotFound + return errors.As(err, ¬Found) + }, 20*time.Second, 200*time.Millisecond) +} + +// A wake on a Stream of a first run that continued as new ends at its wait, since the Stream's +// tasks run under the closed run. +func (s *StreamTestSuite) TestWakeInAClosedFirstRunEndsAtItsWait() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + owned := s.closeWithRun(sc, env, c) + c.respond(c.continueAsNew()) + + reader := s.newCaller(env) + var woke streamservice.ReadRecordsResponse + failure, started := reader.outcome(reader.schedule(streamOp{"ReadRecords", + &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, Wait: durationpb.New(2 * time.Second), + }})[0], &woke) + s.Nil(failure) + s.True(started) + s.Empty(woke.GetRecords()) + s.False(woke.GetDone()) +} + +// The readers check runs in a first run that continued as new, so the floor of a reader that +// closed is dropped, and the Retention trims its range. +func (s *StreamTestSuite) TestReadersCheckRunsInAClosedFirstRun() { + env := s.newEnv(testcore.WithDynamicConfig(stream.ReaderCheckInterval, time.Second)) + sc := s.streamClient(env) + c := s.newCaller(env) + readerRun := s.newCaller(env).run + owned, appended := s.createWithRetention(sc, env, c) + page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned}, + readerLink(env, readerRun)) + s.Nil(failure) + s.Equal([]string{"a"}, kindsAndTopics(page)) + c.respond(c.continueAsNew()) + + // Past the Retention, the floor still keeps the range. + time.Sleep(2 * time.Second) //nolint:forbidigo // The Retention runs on real time. + again, failure := s.directRead(env, &streamservice.ReadRecordsRequest{ + Stream: owned, ToOffset: page.GetNextOffset(), + }) + s.Nil(failure) + s.Equal(page.GetRecords(), again.GetRecords()) + s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), readerRun.GetID(), + readerRun.GetRunID(), "reader done")) + + s.AwaitTrue(func() bool { + _, failure := s.directRead(env, &streamservice.ReadRecordsRequest{ + Stream: owned, FromOffset: appended.GetFirstOffset(), + }) + return failure != nil && causeType(failure) == stream.ReasonExpired + }, 10*time.Second, 200*time.Millisecond) } // An append that names a Workflow Stream by its bare id, or a Stream that doesn't exist, fails. @@ -816,7 +1412,7 @@ func (s *StreamTestSuite) TestAppendToAStreamThatIsNotThereFails() { s.Empty(page.GetRecords()) } -// A Workflow Stream that no Workflow reads trims past the size cap, so its writes go on. +// A Workflow Stream that no Workflow reads trims past the size cap, so its writes don't fail. func (s *StreamTestSuite) TestWorkflowStreamWithNoReaderTrimsPastTheCap() { env := s.newEnv() sc := s.streamClient(env) @@ -826,7 +1422,9 @@ func (s *StreamTestSuite) TestWorkflowStreamWithNoReaderTrimsPastTheCap() { var first int64 for sequence := int64(1); sequence <= 5; sequence++ { resp, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: env.Namespace().String(), Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), + Namespace: env.Namespace(). + String(), + Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), Sequence: sequence, Records: []*streamservice.AppendRecord{ {Topic: "big", Body: &commonpb.Payload{Data: make([]byte, 1024*1024)}}, }, @@ -840,7 +1438,10 @@ func (s *StreamTestSuite) TestWorkflowStreamWithNoReaderTrimsPastTheCap() { _, err := sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{ Namespace: env.Namespace().String(), Stream: owned, FromOffset: first, }) - s.Equal(stream.ReasonExpired, status.Convert(err).Details()[0].(*streamservice.StreamErrorDetails).GetReason()) + s.Equal( + stream.ReasonExpired, + status.Convert(err).Details()[0].(*streamservice.StreamErrorDetails).GetReason(), + ) } // The range that an open Workflow run read stays past the Retention, so its replay reads the same @@ -851,35 +1452,51 @@ func (s *StreamTestSuite) TestWorkflowReaderKeepsItsRangePastTheRetention() { c := s.newCaller(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - owned := &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID()} + owned := &streamservice.StreamRef{ + Id: id, + WorkflowId: c.run.GetID(), + FirstExecutionRunId: c.run.GetRunID(), + } _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ Namespace: ns, StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), Owner: workflowWriter(c.run.GetRunID(), 3), }) s.NoError(err) appended, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: owned, Producer: clientProducer(), Sequence: 1, Records: records("a"), + Namespace: ns, Stream: owned, Producer: clientProducer(), Sequence: 1, + Records: records("a"), }) s.NoError(err) readerRun := s.newCaller(env).run - reader := &commonpb.Link{Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: ns, WorkflowId: readerRun.GetID(), RunId: readerRun.GetRunID(), - }}} + reader := &commonpb.Link{ + Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ + Namespace: ns, WorkflowId: readerRun.GetID(), RunId: readerRun.GetRunID(), + }}, + } page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned}, reader) s.Nil(failure) s.Equal([]string{"a"}, kindsAndTopics(page)) // Past the Retention, and past a reader check, the replay reads the same page. time.Sleep(3 * time.Second) //nolint:forbidigo // The Retention and the check run on real time. - again, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, ToOffset: page.GetNextOffset()}) + again, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, ToOffset: page.GetNextOffset()}, + ) s.Nil(failure) s.Equal(page.GetRecords(), again.GetRecords()) s.Equal(page.GetNextOffset(), again.GetNextOffset()) - s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), readerRun.GetID(), readerRun.GetRunID(), "reader done")) + s.NoError( + env.SdkClient(). + TerminateWorkflow(s.Context(), readerRun.GetID(), readerRun.GetRunID(), "reader done"), + ) s.AwaitTrue(func() bool { - _, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: appended.GetFirstOffset()}) + _, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: appended.GetFirstOffset()}, + ) return failure != nil && causeType(failure) == stream.ReasonExpired }, 10*time.Second, 200*time.Millisecond) } @@ -891,14 +1508,19 @@ func (s *StreamTestSuite) TestReadOfADroppedRangeFailsAsExpired() { sc := s.streamClient(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) var first int64 for sequence := int64(1); sequence <= 3; sequence++ { resp, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: sequence, - Records: []*streamservice.AppendRecord{{Topic: "big", Body: &commonpb.Payload{Data: make([]byte, 1536*1024)}}}, + Records: []*streamservice.AppendRecord{ + {Topic: "big", Body: &commonpb.Payload{Data: make([]byte, 1536*1024)}}, + }, }) s.NoError(err) if first == 0 { @@ -906,13 +1528,19 @@ func (s *StreamTestSuite) TestReadOfADroppedRangeFailsAsExpired() { } } - _, err = sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id), FromOffset: first}) + _, err = sc.ReadRecords( + s.Context(), + &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id), FromOffset: first}, + ) st := status.Convert(err) s.Equal(codes.NotFound, st.Code()) s.Equal(stream.ReasonExpired, st.Details()[0].(*streamservice.StreamErrorDetails).GetReason()) // A direct page answers the reason in its operation failure. - _, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: ref(id), FromOffset: first}) + _, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: ref(id), FromOffset: first}, + ) s.Equal(stream.ReasonExpired, causeType(failure)) } @@ -922,7 +1550,10 @@ func (s *StreamTestSuite) TestDirectReadErrorsKeepTheirReason() { env := s.newEnv() sc := s.streamClient(env) id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: env.Namespace().String(), StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: env.Namespace().String(), StreamId: id}, + ) s.NoError(err) for reason, req := range map[string]*streamservice.ReadRecordsRequest{ @@ -943,14 +1574,20 @@ func (s *StreamTestSuite) createWithRetention( c *streamCaller, ) (*streamservice.StreamRef, *streamservice.AppendRecordsResponse) { id := testcore.RandomizeStr("chat") - owned := &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID()} + owned := &streamservice.StreamRef{ + Id: id, + WorkflowId: c.run.GetID(), + FirstExecutionRunId: c.run.GetRunID(), + } + ns := env.Namespace().String() _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ - Namespace: env.Namespace().String(), StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), + Namespace: ns, StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), Owner: workflowWriter(c.run.GetRunID(), 3), }) s.NoError(err) appended, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: env.Namespace().String(), Stream: owned, Producer: clientProducer(), Sequence: 1, Records: records("a"), + Namespace: ns, Stream: owned, Producer: clientProducer(), Sequence: 1, + Records: records("a"), }) s.NoError(err) return owned, appended @@ -965,13 +1602,19 @@ func (s *StreamTestSuite) TestWorkflowWakeKeepsItsFloor() { reader := s.newCaller(env) var woke streamservice.ReadRecordsResponse - failure, _ := reader.outcome(reader.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ - Stream: owned, WakeOnly: true, - }})[0], &woke) + failure, _ := reader.outcome( + reader.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, + }})[0], + &woke, + ) s.Nil(failure) time.Sleep(3 * time.Second) //nolint:forbidigo // The Retention runs on real time. - page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: woke.GetNextOffset()}) + page, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: woke.GetNextOffset()}, + ) s.Nil(failure) s.Equal([]string{"a"}, kindsAndTopics(page)) } @@ -982,15 +1625,20 @@ func (s *StreamTestSuite) TestReadLinkToNoRunKeepsNoFloor() { sc := s.streamClient(env) owned, appended := s.createWithRetention(sc, env, s.newCaller(env)) - forged := &commonpb.Link{Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: env.Namespace().String(), WorkflowId: "nobody", RunId: uuid.NewString(), - }}} + forged := &commonpb.Link{ + Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ + Namespace: env.Namespace().String(), WorkflowId: "nobody", RunId: uuid.NewString(), + }}, + } page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned}, forged) s.Nil(failure) s.Equal([]string{"a"}, kindsAndTopics(page)) time.Sleep(3 * time.Second) //nolint:forbidigo // The Retention runs on real time. - _, failure = s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: appended.GetFirstOffset()}) + _, failure = s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: appended.GetFirstOffset()}, + ) s.Equal(stream.ReasonExpired, causeType(failure)) } @@ -1001,16 +1649,21 @@ func (s *StreamTestSuite) TestDirectReadPastTheReaderCapIsResourceExhausted() { owned, _ := s.createWithRetention(sc, env, s.newCaller(env)) reader := s.newCaller(env).run - _, err := env.FrontendClient().StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ - Namespace: env.Namespace().String(), - Endpoint: commonnexus.SystemEndpoint, - Service: stream.ServiceName, - Operation: "ReadRecords", - Input: payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{Stream: owned}), - Links: []*commonpb.Link{{Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: env.Namespace().String(), WorkflowId: reader.GetID(), RunId: reader.GetRunID(), - }}}}, - }) + _, err := env.FrontendClient(). + StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ + Namespace: env.Namespace().String(), + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: "ReadRecords", + Input: payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{Stream: owned}), + Links: []*commonpb.Link{ + {Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ + Namespace: env.Namespace(). + String(), + WorkflowId: reader.GetID(), RunId: reader.GetRunID(), + }}}, + }, + }) var exhausted *serviceerror.ResourceExhausted s.ErrorAs(err, &exhausted) s.Contains(exhausted.Error(), "most Workflow readers") @@ -1050,6 +1703,53 @@ func (s *StreamTestSuite) TestReaderThatKeepsHalfTheCapIsToldToContinueAsNew() { s.True(page.GetContinueAsNewSuggested()) } +// A Workflow reader of a client's Stream keeps its floor too. An append that the size cap would +// make drop the reader's range fails, and the reader's replay reads the same page. +func (s *StreamTestSuite) TestWorkflowReaderOfAClientStreamKeepsItsRangePastTheCap() { + env := s.newEnv(testcore.WithDynamicConfig(stream.MaxStoredBytes, 4096)) + sc := s.streamClient(env) + ns := env.Namespace().String() + id := testcore.RandomizeStr("chat") + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) + s.NoError(err) + _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("a"), + }) + s.NoError(err) + + reader := readerLink(env, s.newCaller(env).run) + page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: ref(id)}, reader) + s.Nil(failure) + s.Equal([]string{"a"}, kindsAndTopics(page)) + + bigAppend := func(sequence int64) error { + _, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: sequence, + Records: []*streamservice.AppendRecord{ + {Topic: "big", Body: &commonpb.Payload{Data: make([]byte, 2500)}}, + }, + }) + return err + } + s.NoError(bigAppend(2)) + s.Equal(stream.ReasonTooLarge, reasonOf(bigAppend(3))) + + again, failure := s.directRead(env, &streamservice.ReadRecordsRequest{ + Stream: ref(id), ToOffset: page.GetNextOffset(), + }) + s.Nil(failure) + s.Equal(page.GetRecords(), again.GetRecords()) + next, failure := s.directRead(env, &streamservice.ReadRecordsRequest{ + Stream: ref(id), FromOffset: page.GetNextOffset(), + }, reader) + s.Nil(failure) + s.True(next.GetContinueAsNewSuggested()) +} + // A reset run's first live page gives the lowest offset its copied reads read, so its floor // keeps them and its replay finds them past the Retention. func (s *StreamTestSuite) TestFloorFromKeepsTheReadsOfAResetRun() { @@ -1099,8 +1799,8 @@ func localActivityWriter(runID string, started int64) *streamservice.Producer { } } -// A caller with write access can't close a Workflow Stream, write as a run outside its chain, -// fence a run it doesn't follow, or put a Stream into another Workflow's first run. +// A caller with write access can't close a Workflow Stream, write or fence as another run, or +// create a Workflow Stream outside its chain. A Local Activity of the chain can't close it. func (s *StreamTestSuite) TestForgedOwnerCallsAreRefused() { env := s.newEnv() sc := s.streamClient(env) @@ -1116,6 +1816,11 @@ func (s *StreamTestSuite) TestForgedOwnerCallsAreRefused() { }) s.Equal(stream.ReasonNotOwner, reasonOf(err), "a client's close") + _, err = sc.CloseStream(ctx, &streamservice.CloseStreamRequest{ + Namespace: ns, Stream: owned, Producer: localActivityWriter(run, 3), + }) + s.Equal(stream.ReasonNotOwner, reasonOf(err), "a Local Activity's close") + stranger := s.newCaller(env).run.GetRunID() _, err = sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ Namespace: ns, Stream: owned, Producer: localActivityWriter(stranger, 3), Sequence: 1, @@ -1130,6 +1835,10 @@ func (s *StreamTestSuite) TestForgedOwnerCallsAreRefused() { s.Equal(stream.ReasonNotOwner, reasonOf(err), "a withdraw_run of a run it doesn't follow") id := testcore.RandomizeStr("chat") + _, err = sc.CreateStream(ctx, &streamservice.CreateStreamRequest{ + Namespace: ns, StreamId: id, Owner: workflowWriter(run, 3), + }) + s.Equal(stream.ReasonInvalid, reasonOf(err), "a Workflow owner with no Workflow ref") inChain := &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: run} for name, owner := range map[string]*streamservice.Producer{ "a create from another chain": workflowWriter(stranger, 3),