From 8913e3a8af8f70b0e59f9635ca7029fa61058da1 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:46:16 -0700 Subject: [PATCH 01/23] Closed a Workflow Stream from a System Nexus command of its chain. The server starts the operation only once the Workflow Task that issued it commits, so a failed task closes nothing. The caller is the run in the operation's link, and the close goes in as its Workflow Producer with no epoch. --- chasm/lib/stream/handler.go | 41 +++++++++++++ chasm/lib/stream/nexus.go | 14 ++++- chasm/lib/stream/nexus_test.go | 7 +++ chasm/lib/stream/stream.go | 26 +++++--- chasm/lib/stream/stream_test.go | 48 +++++++++++++++ tests/stream_test.go | 102 ++++++++++++++++++++++++++++---- 6 files changed, 217 insertions(+), 21 deletions(-) diff --git a/chasm/lib/stream/handler.go b/chasm/lib/stream/handler.go index 14d980a5ad3..21f313902cc 100644 --- a/chasm/lib/stream/handler.go +++ b/chasm/lib/stream/handler.go @@ -276,6 +276,47 @@ func (h *handler) CloseStream( return resp, streamError(ctx, t, err) } +// CloseFromRun closes a Workflow Stream for the System Nexus command of run, the calling run +// from the link that the server's Nexus call adds. Run must be an open run of the Stream's chain, +// and the close goes in as run's Workflow Producer. Only the result, the failure and the status +// come from the request. +func (h *handler) CloseFromRun( + ctx context.Context, + req *streamservice.CloseStreamRequest, + run *ReaderRun, +) (*streamservice.CloseStreamResponse, error) { + t, limits, err := h.target(req.GetNamespace(), req.GetStream().GetId(), req.GetStream()) + if err != nil { + return nil, err + } + if run == nil { + return nil, notOwner("A CloseStream through System Nexus needs the link of its calling run") + } + if !t.workflow() { + return nil, notOwner("Only a Workflow Stream closes through System Nexus") + } + command := &streamservice.CloseStreamRequest{ + Namespace: req.GetNamespace(), + Stream: req.GetStream(), + Producer: &streamservice.Producer{ + Kind: streamservice.PRODUCER_KIND_WORKFLOW, + Id: "wf:" + encodePart(run.RunID), + RunId: run.RunID, + }, + Result: req.GetResult(), + Failure: req.GetFailure(), + Status: req.GetStatus(), + } + if err := h.checkChain(ctx, t, command.GetProducer(), ""); err != nil { + return nil, err + } + resp, err := update(ctx, t, nil, nil, + func(s *Stream, mctx chasm.MutableContext) (*streamservice.CloseStreamResponse, error) { + return s.Close(mctx, CloseInput{Request: command, Limits: limits, Command: true}) + }) + return resp, streamError(ctx, t, err) +} + // DescribeStream answers the state of a Stream. func (h *handler) DescribeStream( ctx context.Context, diff --git a/chasm/lib/stream/nexus.go b/chasm/lib/stream/nexus.go index f40af5d22a8..93ec446a74a 100644 --- a/chasm/lib/stream/nexus.go +++ b/chasm/lib/stream/nexus.go @@ -50,8 +50,9 @@ func nexusError(err error) error { return err } -// newNexusService serves the reads to Workflow code. Workflow writes go over the frontend from -// the Core extension, which knows the Workflow Task facts of each write. +// newNexusService serves the reads and the close to Workflow code. The close is a command, so the +// server runs it only when the Workflow Task that issued it commits. Workflow writes go over the +// frontend from the Core extension, which knows the Workflow Task facts of each write. func newNexusService(h *handler) *nexus.Service { service := nexus.NewService(ServiceName) service.MustRegister( @@ -60,6 +61,11 @@ func newNexusService(h *handler) *nexus.Service { resp, err := h.DescribeStream(ctx, req) return resp, nexusError(err) }), + nexus.NewSyncOperation("CloseStream", + func(ctx context.Context, req *streamservice.CloseStreamRequest, options nexus.StartOperationOptions) (*streamservice.CloseStreamResponse, error) { + resp, err := h.CloseFromRun(ctx, req, readerRunOf(options.Links)) + return resp, nexusError(err) + }), &readRecordsOperation{handler: h}, ) return service @@ -222,5 +228,9 @@ func newNexusServiceProcessor(config *Config) *chasm.NexusServiceProcessor { config: config, setNamespace: func(r *streamservice.DescribeStreamRequest, ns string) { r.Namespace = ns }, })) + sp.MustRegisterOperation("CloseStream", chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.CloseStreamRequest]{ + config: config, + setNamespace: func(r *streamservice.CloseStreamRequest, ns string) { r.Namespace = ns }, + })) return sp } diff --git a/chasm/lib/stream/nexus_test.go b/chasm/lib/stream/nexus_test.go index a7be7863182..701a7fe6559 100644 --- a/chasm/lib/stream/nexus_test.go +++ b/chasm/lib/stream/nexus_test.go @@ -46,6 +46,13 @@ func TestProcessorRoutesByStreamID(t *testing.T) { require.NoError(t, payloads.Decode(&commonpb.Payloads{Payloads: []*commonpb.Payload{result.ReserializedInputPayload}}, &routed)) require.Equal(t, "ns", routed.GetNamespace()) + // A Workflow Stream is on its Workflow's shard. + result, err = sp.ProcessInput(ctx, "CloseStream", payloads.MustEncodeSingle(&streamservice.CloseStreamRequest{ + Stream: &streamservice.StreamRef{Id: "chat-1", WorkflowId: "wf", FirstExecutionRunId: "first"}, + })) + require.NoError(t, err) + require.Equal(t, chasm.NexusOperationRoutingKeyExecution{NamespaceID: "ns-id", BusinessID: "wf"}, result.RoutingKey) + _, err = sp.ProcessInput(ctx, "ReadRecords", payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{})) require.ErrorContains(t, err, ReasonInvalid) _, err = sp.ProcessInput(ctx, "ReadRecords", payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{ diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index c074d8994a1..2d02f6ac0da 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -363,6 +363,10 @@ func (s *Stream) checkSize(ctx chasm.MutableContext, size int64, limits Limits) type CloseInput struct { Request *streamservice.CloseStreamRequest Limits Limits + // Command is a Workflow run's System Nexus close, which the server runs once the Workflow + // Task that issued it commits. Its Producer has no epoch, so it fences nothing and withdraws + // nothing. + Command bool } // Close writes the close Record, with the Producer checks and the withdrawal of a batch. A @@ -384,15 +388,21 @@ func (s *Stream) Close(ctx chasm.MutableContext, in CloseInput) (*streamservice. if err := s.checkCloser(producer); err != nil { return nil, err } - w := Withdrawal{PreviousCompleted: req.GetPreviousCompletedTask(), FailedTasks: req.GetFailedTasks(), WithdrawRun: req.GetWithdrawRun()} - flagged, err := s.admit(ctx, producer, w) - if err != nil { - return nil, err - } - if flagged { - s.write(ctx, []*streamservice.StreamRecord{supersededRecord(producer, w)}) + if in.Command { + if s.runFenced(producer.GetRunId()) { + return nil, NewError(ReasonFenced, "A Workflow retry withdrew the run of the Producer") + } + } else { + w := Withdrawal{PreviousCompleted: req.GetPreviousCompletedTask(), FailedTasks: req.GetFailedTasks(), WithdrawRun: req.GetWithdrawRun()} + flagged, err := s.admit(ctx, producer, w) + if err != nil { + return nil, err + } + if flagged { + s.write(ctx, []*streamservice.StreamRecord{supersededRecord(producer, w)}) + } + s.settle(ctx, producer, w, false) } - s.settle(ctx, producer, w, false) s.close(ctx, producer, req.GetResult(), req.GetFailure(), status) s.finish(ctx, in.Limits) return &streamservice.CloseStreamResponse{}, nil diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index 2c68f1334a8..0b0982f4c42 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -21,6 +21,7 @@ import ( "go.temporal.io/server/common/clock" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/testing/protorequire" "google.golang.org/grpc/codes" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" @@ -95,6 +96,19 @@ func (h *harness) close(result string) error { return err } +// commandClose closes the Stream with the System Nexus command of run. +func (h *harness) commandClose(run, result string) error { + _, _, err := chasm.UpdateComponent(h.ctx, h.ref(), (*Stream).Close, CloseInput{ + Request: &streamservice.CloseStreamRequest{ + Producer: &streamservice.Producer{Kind: streamservice.PRODUCER_KIND_WORKFLOW, Id: "wf:" + run, RunId: run}, + Result: &commonpb.Payload{Data: []byte(result)}, + }, + Limits: h.limits, + Command: true, + }) + return err +} + func (h *harness) read(req *streamservice.ReadRecordsRequest) (*streamservice.ReadRecordsResponse, error) { return chasm.ReadComponent(h.ctx, h.ref(), func(s *Stream, ctx chasm.Context, req *streamservice.ReadRecordsRequest) (*streamservice.ReadRecordsResponse, error) { @@ -286,6 +300,40 @@ func TestCloseEndsEveryRead(t *testing.T) { require.Equal(t, CreateOutput{Status: streamservice.STREAM_STATUS_CLOSED}, out) } +// A command close has no epoch, so the epoch of the run's earlier writes doesn't fence it. +func TestCommandCloseOfTheOwnerRun(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Owner: workflowProducer(1)}) + require.NoError(t, err) + _, err = h.append(workflowProducer(3), 1, "a") + require.NoError(t, err) + require.NoError(t, h.commandClose("run", "done")) + require.NoError(t, h.commandClose("run", "again")) + + all, err := h.read(&streamservice.ReadRecordsRequest{}) + require.NoError(t, err) + require.Equal(t, []string{"a", "close"}, topics(all)) + closed := all.GetRecords()[1] + require.Equal(t, "done", string(closed.GetCloseResult().GetData())) + protorequire.ProtoEqual(t, &streamservice.Producer{ + Kind: streamservice.PRODUCER_KIND_WORKFLOW, + Id: "wf:run", + RunId: "run", + }, closed.GetProducer()) +} + +func TestCommandCloseOfAWithdrawnRunIsFenced(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Owner: workflowProducer(1)}) + require.NoError(t, err) + retry := workflowProducer(3) + retry.Id, retry.RunId = "wf:retry", "retry" + _, err = h.appendRequest(&streamservice.AppendRecordsRequest{Producer: retry, Sequence: 1, Records: records("a"), WithdrawRun: "run"}) + require.NoError(t, err) + + require.ErrorContains(t, h.commandClose("run", "done"), ReasonFenced) +} + func TestRetentionDropsOldRecordsAndExpiresTheStream(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Retention: durationpb.New(time.Hour)}) diff --git a/tests/stream_test.go b/tests/stream_test.go index babba2117a4..d2f1cf597f2 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -92,17 +92,7 @@ func (c *streamCaller) schedule(ops ...streamOp) []int64 { last := c.history[len(c.history)-1].GetEventId() commands := make([]*commandpb.Command, 0, len(ops)) for _, op := range ops { - commands = append(commands, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, - Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ - ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ - Endpoint: commonnexus.SystemEndpoint, - Service: stream.ServiceName, - Operation: op.name, - Input: payloads.MustEncodeSingle(op.request), - }, - }, - }) + commands = append(commands, nexusCommand(op)) } c.respond(commands...) var ids []int64 @@ -157,6 +147,21 @@ type streamOp struct { request proto.Message } +// nexusCommand schedules op as a System Nexus operation. +func nexusCommand(op streamOp) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: op.name, + Input: payloads.MustEncodeSingle(op.request), + }, + }, + } +} + func ref(id string) *streamservice.StreamRef { return &streamservice.StreamRef{Id: id} } @@ -502,6 +507,81 @@ func (s *StreamTestSuite) TestStreamClosesWithATerminatedOwner() { s.Equal(streamservice.CLOSE_STATUS_TERMINATED, closed.GetCloseStatus()) } +// closeOp closes a Workflow Stream through System Nexus, as the calling run. +func closeOp(owned *streamservice.StreamRef, result string) streamOp { + return streamOp{"CloseStream", &streamservice.CloseStreamRequest{Stream: owned, Result: payload.EncodeString(result)}} +} + +func (s *StreamTestSuite) TestWorkflowCloseCommandClosesItsStream() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + owned := s.closeWithRun(sc, env, c) + + s.Nil(c.call(closeOp(owned, "answer"), &streamservice.CloseStreamResponse{})) + + closed := s.readClose(sc, env, owned) + s.Equal(payload.EncodeString("answer").GetData(), closed.GetCloseResult().GetData()) + s.Equal(streamservice.CLOSE_STATUS_COMPLETED, closed.GetCloseStatus()) + s.Equal("wf:"+stream.EncodeIDPart(c.run.GetRunID()), closed.GetProducer().GetId()) +} + +// A run of another chain can't close the Stream, and neither can a call with no link to its run. +func (s *StreamTestSuite) TestCloseCommandOfAnotherChainIsRefused() { + env := s.newEnv() + sc := s.streamClient(env) + ns := env.Namespace().String() + owned := s.closeWithRun(sc, env, s.newCaller(env)) + + failure := s.newCaller(env).call(closeOp(owned, "answer"), &streamservice.CloseStreamResponse{}) + s.Equal(stream.ReasonNotOwner, causeType(failure)) + + resp, err := env.FrontendClient().StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ + Namespace: ns, + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: "CloseStream", + Input: payloads.MustEncodeSingle(closeOp(owned, "answer").request), + }) + s.NoError(err) + s.Equal(stream.ReasonNotOwner, causeType(resp.GetOperationFailure().GetFailure())) + + described, err := sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{Namespace: ns, Stream: owned}) + s.NoError(err) + s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) +} + +// A bad command fails the Workflow Task, so the server drops the close command that came with it. +// The retried task's close command closes the Stream once. +func (s *StreamTestSuite) TestCloseCommandOfAFailedWorkflowTaskIsDropped() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + owned := s.closeWithRun(sc, env, c) + + _, err := env.FrontendClient().RespondWorkflowTaskCompleted(s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Identity: "test", + TaskToken: c.token, + Commands: []*commandpb.Command{ + nexusCommand(closeOp(owned, "failed task")), + { + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{}, + }, + }, + }, + }) + s.Error(err) + c.poll() + + s.Nil(c.call(closeOp(owned, "retried task"), &streamservice.CloseStreamResponse{})) + page, err := sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: owned}) + s.NoError(err) + s.Equal([]string{"close"}, kindsAndTopics(page)) + s.Equal(payload.EncodeString("retried task").GetData(), page.GetRecords()[0].GetCloseResult().GetData()) +} + // A cron run's end isn't the chain's end, so the next cron run writes on. The terminate of a later // run ends the chain and closes the Stream with its status. func (s *StreamTestSuite) TestCronRunEndKeepsTheStreamOpen() { From 7dfc8a77cb1f9913d23199822aeb210774491d51 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 17:25:32 -0700 Subject: [PATCH 02/23] Refused a direct System Nexus CloseStream, so only a command closes. A command's start always has the system callback and a direct call never has one. A direct call waits for no Workflow Task to commit, so it gets stream_invalid and a pointer to the gRPC call. --- chasm/lib/stream/nexus.go | 5 +++++ tests/stream_test.go | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/chasm/lib/stream/nexus.go b/chasm/lib/stream/nexus.go index 93ec446a74a..15b284332a5 100644 --- a/chasm/lib/stream/nexus.go +++ b/chasm/lib/stream/nexus.go @@ -63,6 +63,11 @@ func newNexusService(h *handler) *nexus.Service { }), nexus.NewSyncOperation("CloseStream", func(ctx context.Context, req *streamservice.CloseStreamRequest, options nexus.StartOperationOptions) (*streamservice.CloseStreamResponse, error) { + // A command's start always has the system callback, and a direct call never has + // one. A direct call waits for no Workflow Task to commit, so it can't close. + if options.CallbackURL == "" { + return nil, nexusError(invalid("A direct System Nexus call can't close a Stream, so use the gRPC CloseStream")) + } resp, err := h.CloseFromRun(ctx, req, readerRunOf(options.Links)) return resp, nexusError(err) }), diff --git a/tests/stream_test.go b/tests/stream_test.go index d2f1cf597f2..69e896c4d81 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -526,25 +526,38 @@ func (s *StreamTestSuite) TestWorkflowCloseCommandClosesItsStream() { s.Equal("wf:"+stream.EncodeIDPart(c.run.GetRunID()), closed.GetProducer().GetId()) } -// A run of another chain can't close the Stream, and neither can a call with no link to its run. func (s *StreamTestSuite) TestCloseCommandOfAnotherChainIsRefused() { env := s.newEnv() sc := s.streamClient(env) - ns := env.Namespace().String() owned := s.closeWithRun(sc, env, s.newCaller(env)) failure := s.newCaller(env).call(closeOp(owned, "answer"), &streamservice.CloseStreamResponse{}) s.Equal(stream.ReasonNotOwner, causeType(failure)) + described, err := sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{Namespace: env.Namespace().String(), Stream: owned}) + s.NoError(err) + s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) +} + +// A direct System Nexus call waits for no Workflow Task to commit, so it can't close the Stream, +// even with the link of the owner run. +func (s *StreamTestSuite) TestDirectCloseCallIsRefused() { + env := s.newEnv() + sc := s.streamClient(env) + ns := env.Namespace().String() + c := s.newCaller(env) + owned := s.closeWithRun(sc, env, c) + resp, err := env.FrontendClient().StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ Namespace: ns, Endpoint: commonnexus.SystemEndpoint, Service: stream.ServiceName, Operation: "CloseStream", Input: payloads.MustEncodeSingle(closeOp(owned, "answer").request), + Links: []*commonpb.Link{readerLink(env, c.run)}, }) s.NoError(err) - s.Equal(stream.ReasonNotOwner, causeType(resp.GetOperationFailure().GetFailure())) + s.Equal(stream.ReasonInvalid, causeType(resp.GetOperationFailure().GetFailure())) described, err := sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{Namespace: ns, Stream: owned}) s.NoError(err) From f64bf92c743b52326217e21d69ebf02b783aefb4 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 00:59:35 -0700 Subject: [PATCH 03/23] Refused a Local Activity close of a Workflow Stream. A Local Activity writes before its Workflow Task commits, and a failed task can't take the close back. So only the close Activity or the System Nexus command closes. --- chasm/lib/stream/owner.go | 8 +++++--- chasm/lib/stream/stream_test.go | 17 +++++++++++++++++ tests/stream_test.go | 8 +++++++- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/chasm/lib/stream/owner.go b/chasm/lib/stream/owner.go index d9769535a38..0c1239b9d64 100644 --- a/chasm/lib/stream/owner.go +++ b/chasm/lib/stream/owner.go @@ -52,14 +52,16 @@ func checkOwner(owner *streamservice.Producer, t target) error { } // checkCloser refuses a close that the Stream's owner doesn't allow. A Workflow Stream closes -// only from its run chain, and a client's Stream only from a client. The server's own close with -// the run chain comes from no caller. +// only as a run of its chain, and a client's Stream only from a client. The server's own close +// with the run chain comes from no caller. func (s *Stream) checkCloser(producer *streamservice.Producer) error { kind := producer.GetKind() owner := ownerKind(s.GetOwner()) switch owner { case streamservice.PRODUCER_KIND_WORKFLOW: - if ofRun(kind) { + // A Local Activity writes before its Workflow Task commits, and a failed task can't + // take a close back, so only the run's close Activity or command closes. + if kind == streamservice.PRODUCER_KIND_WORKFLOW { return nil } case streamservice.PRODUCER_KIND_CLIENT: diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index 0b0982f4c42..28cae0e80cb 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -322,6 +322,23 @@ func TestCommandCloseOfTheOwnerRun(t *testing.T) { }, closed.GetProducer()) } +// A Local Activity writes before its Workflow Task commits, so it can't close a Workflow Stream. +// A failed task would leave the Stream closed, and the retried task's create would fail. +func TestLocalActivityCantCloseAWorkflowStream(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(1), + }) + require.NoError(t, err) + _, _, err = chasm.UpdateComponent(h.ctx, h.ref(), (*Stream).Close, CloseInput{ + Request: &streamservice.CloseStreamRequest{Producer: localActivityProducer(3)}, + Limits: h.limits, + }) + require.ErrorContains(t, err, ReasonNotOwner) + + require.NoError(t, h.commandClose("run", "done")) +} + func TestCommandCloseOfAWithdrawnRunIsFenced(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Owner: workflowProducer(1)}) diff --git a/tests/stream_test.go b/tests/stream_test.go index 69e896c4d81..2eb48e83ac9 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -1193,7 +1193,8 @@ func localActivityWriter(runID string, started int64) *streamservice.Producer { } // A caller with write access can't close a Workflow Stream, write as a run outside its chain, -// fence a run it doesn't follow, or put a Stream into another Workflow's first run. +// fence a run it doesn't follow, or put a Stream into another Workflow's first run. A Local +// Activity of the chain can't close it either, since it writes before its task commits. func (s *StreamTestSuite) TestForgedOwnerCallsAreRefused() { env := s.newEnv() sc := s.streamClient(env) @@ -1209,6 +1210,11 @@ func (s *StreamTestSuite) TestForgedOwnerCallsAreRefused() { }) s.Equal(stream.ReasonNotOwner, reasonOf(err), "a client's close") + _, err = sc.CloseStream(ctx, &streamservice.CloseStreamRequest{ + Namespace: ns, Stream: owned, Producer: localActivityWriter(run, 3), + }) + s.Equal(stream.ReasonNotOwner, reasonOf(err), "a Local Activity's close") + stranger := s.newCaller(env).run.GetRunID() _, err = sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ Namespace: ns, Stream: owned, Producer: localActivityWriter(stranger, 3), Sequence: 1, From 07db89091c420dcffb575f71352386c6ab1141ee Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:00:03 -0700 Subject: [PATCH 04/23] Checked the closer before a second close of a closed Stream. The crate checks the owner first, so a caller that can't close gets stream_not_owner also after the close. --- chasm/lib/stream/stream.go | 10 +++++++--- chasm/lib/stream/stream_test.go | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index 2d02f6ac0da..d5cdb927116 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -379,15 +379,19 @@ func (s *Stream) Close(ctx chasm.MutableContext, in CloseInput) (*streamservice. return nil, err } s.applyDue(ctx, in.Limits) + // The owner check goes first, as in the crate, so a second close succeeds only for a caller + // that may close. + if s.Exists() { + if err := s.checkCloser(producer); err != nil { + return nil, err + } + } if s.Phase == streampb.STREAM_PHASE_CLOSED { return &streamservice.CloseStreamResponse{}, nil } if err := s.writable(); err != nil { return nil, err } - if err := s.checkCloser(producer); err != nil { - return nil, err - } if in.Command { if s.runFenced(producer.GetRunId()) { return nil, NewError(ReasonFenced, "A Workflow retry withdrew the run of the Producer") diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index 28cae0e80cb..a8289fa7fcb 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -300,6 +300,20 @@ func TestCloseEndsEveryRead(t *testing.T) { require.Equal(t, CreateOutput{Status: streamservice.STREAM_STATUS_CLOSED}, out) } +// A second close does nothing, but only for a caller that may close the Stream. +func TestSecondCloseChecksTheOwner(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + require.NoError(t, h.close("done")) + _, _, err = chasm.UpdateComponent(h.ctx, h.ref(), (*Stream).Close, CloseInput{ + Request: &streamservice.CloseStreamRequest{Producer: activityProducer(1)}, + Limits: h.limits, + }) + require.ErrorContains(t, err, ReasonNotOwner) + require.NoError(t, h.close("again")) +} + // A command close has no epoch, so the epoch of the run's earlier writes doesn't fence it. func TestCommandCloseOfTheOwnerRun(t *testing.T) { h := newHarness(t) From 729d81ba24071ff7073b68a9f548a1fdeb653daf Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:01:33 -0700 Subject: [PATCH 05/23] Refused a Workflow owner outside its chain, and moved the owner. Outside a chain the Stream got the safety TTL with no chain check and no close. A Workflow create of a client Stream now changes the owner too, as the crate does. --- chasm/lib/stream/owner.go | 3 +++ chasm/lib/stream/stream.go | 5 ++++- chasm/lib/stream/stream_test.go | 22 ++++++++++++++++++++++ tests/stream_test.go | 11 ++++++++--- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/chasm/lib/stream/owner.go b/chasm/lib/stream/owner.go index 0c1239b9d64..e855cdb7b23 100644 --- a/chasm/lib/stream/owner.go +++ b/chasm/lib/stream/owner.go @@ -41,6 +41,9 @@ func checkOwner(owner *streamservice.Producer, t target) error { return notOwner("Only the Standalone Activity creates its Stream") case kind == streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY && !t.activity(): return invalid("A Standalone Activity owns only a Stream in its ref") + // Outside a chain the Stream would get the safety TTL with no chain check and no close. + case kind == streamservice.PRODUCER_KIND_WORKFLOW && !t.workflow(): + return invalid("A Workflow owns only a Stream in its ref") case kind == streamservice.PRODUCER_KIND_WORKFLOW || kind == streamservice.PRODUCER_KIND_CLIENT || kind == streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY: diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index d5cdb927116..ab38af83ce1 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -168,8 +168,11 @@ func (s *Stream) Create(ctx chasm.MutableContext, in CreateInput) (CreateOutput, return CreateOutput{}, NewError(ReasonAlreadyExists, "The Stream exists with the Retention %v", s.GetRetention().AsDuration()) } - if owned(in.Request.GetOwner()) { + // The close check reads the owner, so the owner changes with WorkflowOwned, as the crate + // changes both. + if owned(in.Request.GetOwner()) && !s.WorkflowOwned { s.WorkflowOwned = true + s.Owner = in.Request.GetOwner() } s.withdrawOnCreate(ctx, in.Request) s.finish(ctx, in.Limits) diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index a8289fa7fcb..bcc79a69505 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -300,6 +300,28 @@ func TestCloseEndsEveryRead(t *testing.T) { require.Equal(t, CreateOutput{Status: streamservice.STREAM_STATUS_CLOSED}, out) } +// A Workflow owns a Stream only in its chain, where the chain checks and the chain's end apply. +func TestWorkflowOwnerNeedsAWorkflowRef(t *testing.T) { + require.ErrorContains(t, checkOwner(workflowProducer(1), target{}), ReasonInvalid) + nested := target{nested: true, host: chasm.WorkflowArchetypeID} + require.NoError(t, checkOwner(workflowProducer(1), nested)) +} + +// A Workflow's create of an open client Stream makes the Workflow its owner, so a client can't +// close it any more. +func TestWorkflowCreateTakesOverTheOwner(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + _, err = h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(1), + }) + require.NoError(t, err) + + require.ErrorContains(t, h.close("done"), ReasonNotOwner) + require.NoError(t, h.commandClose("run", "done")) +} + // A second close does nothing, but only for a caller that may close the Stream. func TestSecondCloseChecksTheOwner(t *testing.T) { h := newHarness(t) diff --git a/tests/stream_test.go b/tests/stream_test.go index 2eb48e83ac9..e712ff4b131 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -218,7 +218,7 @@ func (s *StreamTestSuite) TestWorkflowReadAndDescribe() { id := testcore.RandomizeStr("chat") writer := workflowWriter("run-1", 3) - _, err := sc.CreateStream(ctx, &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id, Owner: writer}) + _, err := sc.CreateStream(ctx, &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) s.NoError(err) appended, err := sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ Namespace: ns, Stream: ref(id), Producer: writer, Sequence: 1, Records: records("turns", "tools"), @@ -235,7 +235,7 @@ func (s *StreamTestSuite) TestWorkflowReadAndDescribe() { s.Nil(c.call(streamOp{"DescribeStream", &streamservice.DescribeStreamRequest{Stream: ref(id)}}, &described)) s.Equal(appended.GetLastOffset(), described.GetLastOffset()) - _, err = sc.CloseStream(ctx, &streamservice.CloseStreamRequest{Namespace: ns, Stream: ref(id), Producer: writer, Result: payload.EncodeString("done")}) + _, err = sc.CloseStream(ctx, &streamservice.CloseStreamRequest{Namespace: ns, Stream: ref(id), Producer: clientProducer(), Result: payload.EncodeString("done")}) s.NoError(err) var last streamservice.ReadRecordsResponse s.Nil(c.call(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id), FromOffset: page.GetNextOffset()}}, &last)) @@ -1194,7 +1194,8 @@ func localActivityWriter(runID string, started int64) *streamservice.Producer { // A caller with write access can't close a Workflow Stream, write as a run outside its chain, // fence a run it doesn't follow, or put a Stream into another Workflow's first run. A Local -// Activity of the chain can't close it either, since it writes before its task commits. +// Activity of the chain can't close it either, since it writes before its task commits. A +// Workflow owns no Stream outside its chain. func (s *StreamTestSuite) TestForgedOwnerCallsAreRefused() { env := s.newEnv() sc := s.streamClient(env) @@ -1229,6 +1230,10 @@ func (s *StreamTestSuite) TestForgedOwnerCallsAreRefused() { s.Equal(stream.ReasonNotOwner, reasonOf(err), "a withdraw_run of a run it doesn't follow") id := testcore.RandomizeStr("chat") + _, err = sc.CreateStream(ctx, &streamservice.CreateStreamRequest{ + Namespace: ns, StreamId: id, Owner: workflowWriter(run, 3), + }) + s.Equal(stream.ReasonInvalid, reasonOf(err), "a Workflow owner with no Workflow ref") inChain := &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: run} for name, owner := range map[string]*streamservice.Producer{ "a create from another chain": workflowWriter(stranger, 3), From d3053860f0315f5c6636cfb546560f62ebd60e1d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:02:15 -0700 Subject: [PATCH 06/23] Made a create record only its withdraw_run, as the crate does. A create has no task facts, so it withdrew a run's earlier tasks that a last read then hid. Its Superseded Record now has no epoch, as the crate's. --- chasm/lib/stream/producer.go | 3 +- chasm/lib/stream/stream.go | 13 +++++--- chasm/lib/stream/stream_test.go | 56 +++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/chasm/lib/stream/producer.go b/chasm/lib/stream/producer.go index 3abf10f4def..c745d215bfb 100644 --- a/chasm/lib/stream/producer.go +++ b/chasm/lib/stream/producer.go @@ -136,7 +136,8 @@ func (s *Stream) withdrawsRun(ctx chasm.Context, run string) bool { func supersededRecord(producer *streamservice.Producer, w Withdrawal) *streamservice.StreamRecord { epoch := producer.GetEpoch() if w.Defer { - // A deferred batch withdraws only the retry run's previous run, so no epoch goes with it. + // A deferred batch or a create withdraws only the retry run's previous run, so no epoch + // goes with it. epoch = nil } return &streamservice.StreamRecord{ diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index ab38af83ce1..23a6c6f4e63 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -469,13 +469,18 @@ func (s *Stream) writable() error { } } -// withdrawOnCreate applies a retry run's withdraw_run at its first create. +// withdrawOnCreate applies a retry run's withdraw_run at its first create. A create has no task +// facts, so it withdraws no task, and its Superseded Record has no epoch, as the crate's. func (s *Stream) withdrawOnCreate(ctx chasm.MutableContext, req *streamservice.CreateStreamRequest) { - w := Withdrawal{WithdrawRun: req.GetWithdrawRun()} - if s.withdrawsRun(ctx, w.WithdrawRun) { + run := req.GetWithdrawRun() + if run == "" || s.runFenced(run) { + return + } + if s.withdrawsRun(ctx, run) { + w := Withdrawal{WithdrawRun: run, Defer: true} s.write(ctx, []*streamservice.StreamRecord{supersededRecord(req.GetOwner(), w)}) } - s.settle(ctx, req.GetOwner(), w, false) + s.FencedRuns = append(s.FencedRuns, run) } // write gives the entries their offsets and stores them as one batch. diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index bcc79a69505..a26e392818a 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "encoding/json" + "fmt" "os" "testing" "time" @@ -322,6 +323,61 @@ func TestWorkflowCreateTakesOverTheOwner(t *testing.T) { require.NoError(t, h.commandClose("run", "done")) } +// A create has no task facts. A create again in a later task of the same run withdraws none of +// the run's Records, so a last read gives them as a plain read does. +func TestCreateAgainKeepsTheRunsRecordsInLast(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(3), + }) + require.NoError(t, err) + for i, task := range []int64{3, 7} { + call := localActivityProducer(task) + call.Id = fmt.Sprintf("la:run:%d", task) + _, err := h.append(call, int64(i+1), "t") + require.NoError(t, err) + } + _, err = h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(12), + }) + require.NoError(t, err) + + plain, err := h.read(&streamservice.ReadRecordsRequest{}) + require.NoError(t, err) + require.Equal(t, []string{"t", "t"}, topics(plain)) + last, err := chasm.ReadComponent(h.ctx, h.ref(), (*Stream).Last, + &streamservice.ReadRecordsRequest{Last: 10}) + require.NoError(t, err) + require.Equal(t, []string{"t", "t"}, topics(last)) +} + +// A retry run's create withdraws its previous run with a Superseded Record that has no epoch, as +// the crate's, so a reader withdraws no task of the retry run. +func TestCreateWithdrawsThePreviousRunWithNoEpoch(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: workflowProducer(1), + }) + require.NoError(t, err) + _, err = h.append(workflowProducer(3), 1, "a") + require.NoError(t, err) + retry := workflowProducer(5) + retry.Id, retry.RunId = "wf:retry", "retry" + _, err = h.create(&streamservice.CreateStreamRequest{ + StreamId: "chat-1", Owner: retry, WithdrawRun: "run", + }) + require.NoError(t, err) + + read, err := h.read(&streamservice.ReadRecordsRequest{}) + require.NoError(t, err) + require.Equal(t, []string{"a", "superseded"}, topics(read)) + supersession := read.GetRecords()[1].GetSupersession() + require.Nil(t, supersession.GetEpoch()) + require.Equal(t, "run", supersession.GetWithdrawRun()) + _, err = h.append(workflowProducer(9), 2, "late") + require.ErrorContains(t, err, ReasonFenced) +} + // A second close does nothing, but only for a caller that may close the Stream. func TestSecondCloseChecksTheOwner(t *testing.T) { h := newHarness(t) From ea0cdb53b2ddf7f7aa8a5b5070d0f14758c8e4ef Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:02:55 -0700 Subject: [PATCH 07/23] Scanned an empty filtered page past the scan cap to a match. A page stopped after 10,000 entries of other Topics with no Record. A read with no wait took it for the end, and a parked read waited for a later append. --- chasm/lib/stream/read.go | 3 ++- chasm/lib/stream/stream.go | 3 ++- chasm/lib/stream/stream_test.go | 39 +++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/chasm/lib/stream/read.go b/chasm/lib/stream/read.go index 7513fd61d80..b4d49471d9c 100644 --- a/chasm/lib/stream/read.go +++ b/chasm/lib/stream/read.go @@ -101,7 +101,8 @@ func (s *Stream) scan( if end := req.GetToOffset(); end > 0 && entry.GetOffset() >= end { return resp, nil } - if resp.Done || len(resp.Records) >= limit || scanned >= maxScan { + full := len(resp.Records) >= limit || (scanned >= maxScan && len(resp.Records) > 0) + if resp.Done || full { return resp, nil } if entry.GetOffset() >= retained && overBytes(req, resp, entry, size) { diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index 23a6c6f4e63..2897405cce3 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -27,7 +27,8 @@ const ( MaxRecords = 1000 // MaxStreamIDBytes is the longest Stream id. MaxStreamIDBytes = 4096 - // The most entries one page scans, since a Topic filter can skip many. + // The most entries a page with Records scans, since a Topic filter can skip many. An empty + // page scans on to the end, so a reader never takes it for the end with a match left. maxScan = 10 * MaxRecords // An offset is the append time in ms shifted by these bits, plus a sequence, as the Redis // Store packs its entry ids. So the offsets of the two backends sort the same way, and the diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index a26e392818a..a33b465e80f 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -883,6 +883,45 @@ func TestPageStopsAtMaxBytes(t *testing.T) { require.Equal(t, []string{"big"}, topics(second)) } +// skipped is one batch with more entries of another Topic than one page scans, then a match. +func skipped() []string { + out := make([]string, 0, maxScan+2) + for range maxScan + 1 { + out = append(out, "other") + } + return append(out, "turns") +} + +// A filtered page scans past the scan cap while it has no Record, so a read with no wait doesn't +// take an empty page for the end of the Stream. +func TestFilteredPageScansPastTheCapToAMatch(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + _, err = h.append(activityProducer(1), 1, skipped()...) + require.NoError(t, err) + + page, err := h.read(&streamservice.ReadRecordsRequest{Topics: []string{"turns"}}) + require.NoError(t, err) + require.Equal(t, []string{"turns"}, topics(page)) +} + +// A parked read's look scans past the scan cap too, so it wakes with no later append. +func TestParkedReadWakesForAMatchPastTheCap(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{ + Topics: []string{"turns"}, WakeOnly: true, + }) + require.NoError(t, err) + require.Nil(t, resp) + + _, err = h.append(activityProducer(1), 1, skipped()...) + require.NoError(t, err) + require.True(t, h.parked("read-1").GetResolved()) +} + // A last read gives the newest Records that pass the filter, oldest first, and leaves out the // Records of a withdrawn Workflow Task. func TestLastReadsTheNewestRecords(t *testing.T) { From 5cebe2d0765810e5a6ab68b4441901be680bade7 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:08:07 -0700 Subject: [PATCH 08/23] Parked only wake_only reads, so no parked read keeps a page. One append woke every parked read in one transaction and copied its batch into each. That could push the execution past the mutable state limit. Any other Workflow read now answers at once. --- chasm/lib/stream/handler.go | 5 +-- chasm/lib/stream/nexus.go | 6 +-- chasm/lib/stream/parked_read.go | 4 +- chasm/lib/stream/read.go | 16 ++++---- chasm/lib/stream/stream_test.go | 61 +++++++++++++++++++--------- tests/stream_test.go | 70 +++++++++++++++++++++++++++------ 6 files changed, 115 insertions(+), 47 deletions(-) diff --git a/chasm/lib/stream/handler.go b/chasm/lib/stream/handler.go index 21f313902cc..8eb6634f736 100644 --- a/chasm/lib/stream/handler.go +++ b/chasm/lib/stream/handler.go @@ -349,9 +349,8 @@ func (h *handler) applyDue(ctx context.Context, t target, limits Limits) error { return err } -// ReadOrPark answers a Workflow read at once when the Stream has Records for it or ends. -// Otherwise it parks the read and answers nil, and the Stream completes the read's Nexus -// operation later. +// ReadOrPark answers a Workflow read at once, but parks a wake_only read with no Record for it +// and answers nil. The Stream completes the wake's Nexus operation later. func (h *handler) ReadOrPark( ctx context.Context, req *streamservice.ReadRecordsRequest, diff --git a/chasm/lib/stream/nexus.go b/chasm/lib/stream/nexus.go index 15b284332a5..dfe06e2a2f1 100644 --- a/chasm/lib/stream/nexus.go +++ b/chasm/lib/stream/nexus.go @@ -76,9 +76,9 @@ func newNexusService(h *handler) *nexus.Service { return service } -// readRecordsOperation answers a read at once when the Stream has Records for it or ends. Any -// other read is an async operation that the Stream completes when Records come or the Stream -// closes, so a Workflow waits on the server with no timer. +// readRecordsOperation answers a read at once. A wake_only read with no Record for it is an +// async operation that the Stream completes when Records come or the Stream closes, so a +// Workflow waits on the server with no timer. type readRecordsOperation struct { nexus.UnimplementedOperation[*streamservice.ReadRecordsRequest, *streamservice.ReadRecordsResponse] diff --git a/chasm/lib/stream/parked_read.go b/chasm/lib/stream/parked_read.go index 71d9d0ad1d4..97e139dc02f 100644 --- a/chasm/lib/stream/parked_read.go +++ b/chasm/lib/stream/parked_read.go @@ -17,8 +17,8 @@ import ( var errCanceled = errors.New("the read was canceled") -// ParkedRead is a Workflow read that found no Record. It holds the completion callback of the -// read's Nexus operation, and the Stream completes it when the read can answer. +// ParkedRead is a Workflow's wake_only read that found no Record. It holds the completion +// callback of the read's Nexus operation, and the Stream completes it when the read can answer. type ParkedRead struct { chasm.UnimplementedComponent diff --git a/chasm/lib/stream/read.go b/chasm/lib/stream/read.go index b4d49471d9c..a4c7c96ddcd 100644 --- a/chasm/lib/stream/read.go +++ b/chasm/lib/stream/read.go @@ -329,8 +329,9 @@ func StartForRead(ctx chasm.MutableContext, in ReadInput) (*Stream, error) { return s, nil } -// ReadOrPark answers a read that has Records or ends, and parks any other read until the -// Stream can answer it. It answers nil for a parked read. +// ReadOrPark answers a read at once, but parks a wake_only read that finds no Record until the +// Stream can answer it. It answers nil for a parked read. A wake's answer has no Records, so a +// parked read keeps no page, though one append wakes every parked read in one transaction. func (s *Stream) ReadOrPark(ctx chasm.MutableContext, in ReadInput) (*streamservice.ReadRecordsResponse, error) { s.applyDue(ctx, in.Limits) if field, ok := s.Reads[in.RequestID]; ok { @@ -354,12 +355,13 @@ func (s *Stream) ReadOrPark(ctx chasm.MutableContext, in ReadInput) (*streamserv if in.Reader != nil { read.ReaderRunId = in.Reader.RunID } + parks := in.Callback != nil && in.Request.GetWakeOnly() wait, createWait := parkWaits(in.Request, in.Limits) if wait > 0 { read.WaitDeadline = timestamppb.New(now.Add(wait)) } if s.Phase == streampb.STREAM_PHASE_PENDING { - if createWait <= 0 || in.Callback == nil { + if createWait <= 0 || !parks { return nil, notFound() } read.CreateDeadline = timestamppb.New(now.Add(createWait)) @@ -375,7 +377,7 @@ func (s *Stream) ReadOrPark(ctx chasm.MutableContext, in ReadInput) (*streamserv if err != nil { return nil, err } - if found || in.Callback == nil { + if found || !parks { s.finish(ctx, in.Limits) if in.Request.GetWakeOnly() { resp = s.wakeAnswer(from) @@ -499,13 +501,13 @@ func (s *Stream) pruneReads(ctx chasm.MutableContext) { } } -// parkWaits are the waits of a parked read. A wake_only read may wait for days, since it adds -// nothing to History while it waits. Any other read waits as a long poll does. +// parkWaits are the waits of a parked read, which is a wake_only read. It may wait for days, +// since it adds nothing to History while it waits. func parkWaits( req *streamservice.ReadRecordsRequest, limits Limits, ) (wait time.Duration, createWait time.Duration) { - if !req.GetWakeOnly() || limits.MaxWakeWait <= 0 { + if limits.MaxWakeWait <= 0 { return ReadTimeout(req) } return min(max(req.GetWait().AsDuration(), 0), limits.MaxWakeWait), diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index a33b465e80f..c0b4df6c914 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -468,12 +468,14 @@ func TestRetentionDropsOldRecordsAndExpiresTheStream(t *testing.T) { require.Equal(t, CreateOutput{}, out) } +// A wake answers with no Records once one passes its filter, and the reader reads the page. func TestParkedReadWakesOnAppend(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) require.NoError(t, err) - resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{Topics: []string{"turns"}}) + wake := &streamservice.ReadRecordsRequest{Topics: []string{"turns"}, WakeOnly: true} + resp, err := h.readOrPark("read-1", wake) require.NoError(t, err) require.Nil(t, resp) require.False(t, h.parked("read-1").GetResolved()) @@ -486,19 +488,20 @@ func TestParkedReadWakesOnAppend(t *testing.T) { require.NoError(t, err) read := h.parked("read-1") require.True(t, read.GetResolved()) - require.Equal(t, []string{"turns"}, topics(read.GetResponse())) + require.Empty(t, read.GetResponse().GetRecords()) require.Equal(t, []callbackspb.CallbackStatus{callbackspb.CALLBACK_STATUS_SCHEDULED}, read.callbacks) - again, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{Topics: []string{"turns"}}) + again, err := h.readOrPark("read-1", wake) require.NoError(t, err) - require.Equal(t, []string{"turns"}, topics(again)) + require.Empty(t, again.GetRecords()) + require.Equal(t, int64(0), again.GetNextOffset()) } func TestParkedReadWakesOnClose(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) require.NoError(t, err) - resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{}) + resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{WakeOnly: true}) require.NoError(t, err) require.Nil(t, resp) @@ -523,10 +526,12 @@ func TestReadWithRecordsAnswersAtOnce(t *testing.T) { func TestReadWaitsForTheCreate(t *testing.T) { h := newHarness(t) - _, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{}) + _, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{WakeOnly: true}) require.ErrorContains(t, err, ReasonNotFound) - resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{CreateWait: durationpb.New(30 * time.Second)}) + resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{ + WakeOnly: true, CreateWait: durationpb.New(30 * time.Second), + }) require.NoError(t, err) require.Nil(t, resp) _, err = h.append(activityProducer(1), 1, "a") @@ -539,12 +544,14 @@ func TestReadWaitsForTheCreate(t *testing.T) { _, err = h.append(activityProducer(1), 1, "a") require.NoError(t, err) - require.Equal(t, []string{"a"}, topics(h.parked("read-1").GetResponse())) + require.True(t, h.parked("read-1").GetResolved()) } func TestReadForTheCreateTimesOut(t *testing.T) { h := newHarness(t) - resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{CreateWait: durationpb.New(30 * time.Second)}) + resp, err := h.readOrPark("read-1", &streamservice.ReadRecordsRequest{ + WakeOnly: true, CreateWait: durationpb.New(30 * time.Second), + }) require.NoError(t, err) require.Nil(t, resp) @@ -834,23 +841,19 @@ func TestCallerCantCloseAsTerminated(t *testing.T) { require.Equal(t, streamservice.CLOSE_STATUS_COMPLETED, read.GetRecords()[0].GetCloseStatus()) } -// A wake_only read parks for hours, so an idle Workflow reader adds no History each minute. Any -// other parked read still answers an empty page after 60 s. +// A wake_only read parks for hours, so an idle Workflow reader adds no History each minute. func TestWakeParksPastTheLongPollCap(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) require.NoError(t, err) - for id, wakeOnly := range map[string]bool{"wake": true, "page": false} { - resp, err := h.readOrPark(id, &streamservice.ReadRecordsRequest{ - WakeOnly: wakeOnly, Wait: durationpb.New(2 * time.Hour), - }) - require.NoError(t, err) - require.Nil(t, resp) - } + resp, err := h.readOrPark("wake", &streamservice.ReadRecordsRequest{ + WakeOnly: true, Wait: durationpb.New(2 * time.Hour), + }) + require.NoError(t, err) + require.Nil(t, resp) h.advance(2 * time.Minute) require.False(t, h.parked("wake").GetResolved()) - require.True(t, h.parked("page").GetResolved()) _, err = h.append(activityProducer(1), 1, "a") require.NoError(t, err) @@ -858,6 +861,26 @@ func TestWakeParksPastTheLongPollCap(t *testing.T) { require.Empty(t, h.parked("wake").GetResponse().GetRecords()) } +// Only a wake parks. Any other read answers what the Stream has at once, so no parked read +// keeps a page, and one append that wakes many reads doesn't copy its batch into each. +func TestOnlyAWakeParks(t *testing.T) { + h := newHarness(t) + _, err := h.readOrPark("pending", &streamservice.ReadRecordsRequest{ + CreateWait: durationpb.New(time.Minute), + }) + require.ErrorContains(t, err, ReasonNotFound) + + _, err = h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + resp, err := h.readOrPark("page", &streamservice.ReadRecordsRequest{ + Start: streamservice.READ_START_END, Wait: durationpb.New(time.Minute), + }) + require.NoError(t, err) + require.NotNil(t, resp) + require.Empty(t, resp.GetRecords()) + require.Nil(t, h.parked("page")) +} + // A page stops before the Record that would take it past max_bytes, and a larger Record comes // alone, so a reader with a small cap still moves on. func TestPageStopsAtMaxBytes(t *testing.T) { diff --git a/tests/stream_test.go b/tests/stream_test.go index e712ff4b131..93f0a33aab1 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -243,7 +243,8 @@ func (s *StreamTestSuite) TestWorkflowReadAndDescribe() { s.True(last.GetDone()) } -func (s *StreamTestSuite) TestWorkflowReadWaitsForAnAppend() { +// A Workflow's wake parks until a Record passes its filter, and answers with no Records. +func (s *StreamTestSuite) TestWorkflowWakeWaitsForAnAppend() { env := s.newEnv() sc := s.streamClient(env) c := s.newCaller(env) @@ -252,7 +253,9 @@ func (s *StreamTestSuite) TestWorkflowReadWaitsForAnAppend() { _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) s.NoError(err) - read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id), Topics: []string{"turns"}}})[0] + read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: ref(id), Topics: []string{"turns"}, WakeOnly: true, + }})[0] for i, topic := range []string{"other", "turns"} { _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: int64(i + 1), Records: records(topic), @@ -264,10 +267,11 @@ func (s *StreamTestSuite) TestWorkflowReadWaitsForAnAppend() { failure, started := c.outcome(read, &page) s.Nil(failure) s.True(started) - s.Equal([]string{"turns"}, kindsAndTopics(&page)) + s.Empty(page.GetRecords()) + s.False(page.GetDone()) } -func (s *StreamTestSuite) TestWorkflowReadEndsOnClose() { +func (s *StreamTestSuite) TestWorkflowWakeEndsOnClose() { env := s.newEnv() sc := s.streamClient(env) c := s.newCaller(env) @@ -276,7 +280,9 @@ func (s *StreamTestSuite) TestWorkflowReadEndsOnClose() { _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) s.NoError(err) - read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id)}})[0] + read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: ref(id), WakeOnly: true, + }})[0] _, err = sc.CloseStream(s.Context(), &streamservice.CloseStreamRequest{Namespace: ns, Stream: ref(id), Producer: clientProducer()}) s.NoError(err) @@ -284,7 +290,7 @@ func (s *StreamTestSuite) TestWorkflowReadEndsOnClose() { failure, started := c.outcome(read, &page) s.Nil(failure) s.True(started) - s.Equal([]string{"close"}, kindsAndTopics(&page)) + s.Empty(page.GetRecords()) s.True(page.GetDone()) } @@ -421,7 +427,7 @@ func (s *StreamTestSuite) TestClientReadWaitsForTheCreate() { s.NoError(<-done) } -func (s *StreamTestSuite) TestWorkflowReadWaitsForTheCreate() { +func (s *StreamTestSuite) TestWorkflowWakeWaitsForTheCreate() { env := s.newEnv() sc := s.streamClient(env) c := s.newCaller(env) @@ -430,6 +436,7 @@ func (s *StreamTestSuite) TestWorkflowReadWaitsForTheCreate() { read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ Stream: ref(id), + WakeOnly: true, CreateWait: durationpb.New(30 * time.Second), }})[0] _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) @@ -443,7 +450,7 @@ func (s *StreamTestSuite) TestWorkflowReadWaitsForTheCreate() { failure, started := c.outcome(read, &page) s.Nil(failure) s.True(started) - s.Equal([]string{"turns"}, kindsAndTopics(&page)) + s.Empty(page.GetRecords()) } // closeWithRun creates a Stream that closes with the caller's run chain. @@ -667,7 +674,9 @@ func (s *StreamTestSuite) TestDeferredWorkflowBatchIsWithdrawnByAFailedTask() { s.Equal([]int64{3}, page.GetRecords()[1].GetSupersession().GetFailedTasks()) } -func (s *StreamTestSuite) TestWorkflowReadEndsAtItsWait() { +// A Workflow read that isn't a wake never parks. It answers what the Stream has at once, an +// empty page here, so no parked read keeps a page. +func (s *StreamTestSuite) TestWorkflowReadThatIsNotAWakeAnswersAtOnce() { env := s.newEnv() sc := s.streamClient(env) c := s.newCaller(env) @@ -682,15 +691,48 @@ func (s *StreamTestSuite) TestWorkflowReadEndsAtItsWait() { var page streamservice.ReadRecordsResponse failure, started := c.outcome(c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ - Stream: ref(id), Topics: []string{"turns"}, Wait: durationpb.New(time.Second), + Stream: ref(id), Topics: []string{"turns"}, Wait: durationpb.New(time.Minute), }})[0], &page) s.Nil(failure) - s.True(started) + s.False(started) s.Empty(page.GetRecords()) s.False(page.GetDone()) s.Equal(appended.GetLastOffset()+1, page.GetNextOffset()) } +// One large append must not copy its batch into every waiting read, which would push the +// execution past the mutable state limit and expire the Stream. +func (s *StreamTestSuite) TestLargeAppendWithManyReadersStaysUnderTheStateLimit() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + ns := env.Namespace().String() + id := testcore.RandomizeStr("chat") + _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + s.NoError(err) + + ops := make([]streamOp, 0, 5) + for range 5 { + ops = append(ops, streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: ref(id), Start: streamservice.READ_START_END, Wait: durationpb.New(time.Minute), + }}) + } + c.schedule(ops...) + + big := make([]byte, 1800*1024) + _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: []*streamservice.AppendRecord{{Topic: "big", Body: &commonpb.Payload{Data: big}}}, + }) + s.NoError(err) + // The size check runs in the next transition, such as a wake's callback task. + time.Sleep(2 * time.Second) //nolint:forbidigo // The callback tasks run on real time. + described, err := sc.DescribeStream(s.Context(), + &streamservice.DescribeStreamRequest{Namespace: ns, Stream: ref(id)}) + s.NoError(err) + s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) +} + // An append to a Workflow Stream completes the Workflow's parked read in the Workflow's // transaction, since the Stream nests in the Workflow. func (s *StreamTestSuite) TestNestedStreamWakesItsWorkflow() { @@ -699,7 +741,9 @@ func (s *StreamTestSuite) TestNestedStreamWakesItsWorkflow() { c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) - read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: owned}})[0] + read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, + }})[0] _, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ Namespace: env.Namespace().String(), Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), Sequence: 1, Records: records("turns"), @@ -710,7 +754,7 @@ func (s *StreamTestSuite) TestNestedStreamWakesItsWorkflow() { failure, started := c.outcome(read, &page) s.Nil(failure) s.True(started) - s.Equal([]string{"turns"}, kindsAndTopics(&page)) + s.Empty(page.GetRecords()) // The Stream id names no execution of its own. _, err = sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{Namespace: env.Namespace().String(), Stream: ref(owned.GetId())}) From 3162e3cc6da625417253484d540f79f6cdf217b8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:09:39 -0700 Subject: [PATCH 09/23] Reopened an abandoned Workflow Stream node for a create or a wake. A wake whose create wait ended left the node abandoned, and every later create failed. A nested node has no execution of its own to replace. --- chasm/lib/stream/handler.go | 1 + chasm/lib/stream/read.go | 1 + chasm/lib/stream/stream.go | 9 +++++++++ tests/stream_test.go | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/chasm/lib/stream/handler.go b/chasm/lib/stream/handler.go index 8eb6634f736..e3417c7bf33 100644 --- a/chasm/lib/stream/handler.go +++ b/chasm/lib/stream/handler.go @@ -188,6 +188,7 @@ func (h *handler) CreateStream( } out, err = update(ctx, t, newNestedStream, check, func(s *Stream, mctx chasm.MutableContext) (CreateOutput, error) { return s.Create(mctx, in) }) + err = streamError(ctx, t, err) } else { out, err = h.createStandalone(ctx, t, in) } diff --git a/chasm/lib/stream/read.go b/chasm/lib/stream/read.go index a4c7c96ddcd..9c35f841e4a 100644 --- a/chasm/lib/stream/read.go +++ b/chasm/lib/stream/read.go @@ -360,6 +360,7 @@ func (s *Stream) ReadOrPark(ctx chasm.MutableContext, in ReadInput) (*streamserv if wait > 0 { read.WaitDeadline = timestamppb.New(now.Add(wait)) } + s.reopen() if s.Phase == streampb.STREAM_PHASE_PENDING { if createWait <= 0 || !parks { return nil, notFound() diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index 2897405cce3..3eebfc0c164 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -147,6 +147,7 @@ func StartStream(ctx chasm.MutableContext, in CreateInput) (*Stream, error) { // Create opens the Stream, idempotently for one Retention. func (s *Stream) Create(ctx chasm.MutableContext, in CreateInput) (CreateOutput, error) { s.applyDue(ctx, in.Limits) + s.reopen() retention := in.Request.GetRetention().AsDuration() if retention <= 0 { retention = in.Limits.DefaultRetention @@ -187,6 +188,14 @@ func (s *Stream) Create(ctx chasm.MutableContext, in CreateInput) (CreateOutput, } } +// reopen makes an abandoned nested Stream pending again for a create or a wait for the create. +// Only a Stream of its own has an execution that the caller can replace. +func (s *Stream) reopen() { + if s.Nested && s.Phase == streampb.STREAM_PHASE_ABANDONED { + s.Phase = streampb.STREAM_PHASE_PENDING + } +} + // owned says whether a create's owner keeps the Stream open past one Retention with no write: a // Workflow, or a Standalone Activity, which closes the Stream when it ends. func owned(owner *streamservice.Producer) bool { diff --git a/tests/stream_test.go b/tests/stream_test.go index 93f0a33aab1..ea0dbcc3387 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -453,6 +453,43 @@ func (s *StreamTestSuite) TestWorkflowWakeWaitsForTheCreate() { s.Empty(page.GetRecords()) } +// A wake whose wait for the create ends first leaves the Workflow Stream's node abandoned. The +// node has no execution of its own to replace, so a later wake parks again and the create opens +// it. +func (s *StreamTestSuite) TestAbandonedWorkflowStreamCanStillBeCreated() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + ns := env.Namespace().String() + id := testcore.RandomizeStr("chat") + owned := &streamservice.StreamRef{ + Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID(), + } + + failure := c.call(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, + CreateWait: durationpb.New(time.Second), Wait: durationpb.New(time.Hour), + }}, &streamservice.ReadRecordsResponse{}) + s.Equal(stream.ReasonNotFound, causeType(failure)) + + wake := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, CreateWait: durationpb.New(30 * time.Second), + }})[0] + _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ + Namespace: ns, StreamId: id, Owner: workflowWriter(c.current, 3), Ref: owned, + CloseWithRun: true, + }) + s.NoError(err) + _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ + Namespace: ns, Stream: owned, Producer: clientProducer(), Sequence: 1, + Records: records("turns"), + }) + s.NoError(err) + failure, started := c.outcome(wake, &streamservice.ReadRecordsResponse{}) + s.Nil(failure) + s.True(started) +} + // closeWithRun creates a Stream that closes with the caller's run chain. func (s *StreamTestSuite) closeWithRun(sc streamservice.StreamServiceClient, env *testcore.TestEnv, c *streamCaller) *streamservice.StreamRef { id := testcore.RandomizeStr("chat") From c592f73b9e563bbb712a0abe4620851141ac7292 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:12:10 -0700 Subject: [PATCH 10/23] Kept Workflow reader floors on client Streams too. The size cap dropped what a Workflow read from a client's Stream, and its replay then failed in a loop. An append that would drop a floor's range now fails, as on a Workflow Stream. --- chasm/lib/stream/config.go | 11 +++++---- chasm/lib/stream/handler.go | 4 +-- chasm/lib/stream/host.go | 2 +- chasm/lib/stream/readers.go | 8 +++--- chasm/lib/stream/stream.go | 8 +++--- chasm/lib/stream/stream_test.go | 28 +++++++++++++++++++++ tests/stream_test.go | 43 +++++++++++++++++++++++++++++++++ 7 files changed, 88 insertions(+), 16 deletions(-) diff --git a/chasm/lib/stream/config.go b/chasm/lib/stream/config.go index af1f9908d14..8e6e6696953 100644 --- a/chasm/lib/stream/config.go +++ b/chasm/lib/stream/config.go @@ -31,7 +31,8 @@ Stream that a client or an Activity created lives one Retention after its last w "stream.maxStoredBytes", 4*1024*1024, `The most Record bytes a Stream keeps. The Records are in the Stream's mutable state, so an -append past the cap drops the oldest batches, as the Retention does.`, +append past the cap drops the oldest batches, as the Retention does. The append fails when an open +Workflow reader still needs them.`, ) MaxWorkflowStateBytes = dynamicconfig.NewNamespaceIntSetting( @@ -57,15 +58,15 @@ retryable error, so the caller backs off.`, MaxWorkflowReaders = dynamicconfig.NewNamespaceIntSetting( "stream.maxWorkflowReaders", 100, - `The most open Workflow runs whose read floor one Workflow Stream keeps. A page of a new -reader past the cap fails with a retryable error, so the caller backs off.`, + `The most open Workflow runs whose read floor one Stream keeps. A page of a new reader past +the cap fails with a retryable error, so the caller backs off.`, ) ReaderCheckInterval = dynamicconfig.NewNamespaceDurationSetting( "stream.readerCheckInterval", 5*time.Minute, - `How often a Workflow Stream with read floors checks whether their runs are still open. The -floor of a closed run goes at the next check.`, + `How often a Stream with read floors checks whether their runs are still open. The floor of +a closed run goes at the next check.`, ) MaxWakeWait = dynamicconfig.NewNamespaceDurationSetting( diff --git a/chasm/lib/stream/handler.go b/chasm/lib/stream/handler.go index e3417c7bf33..10bf1931bc5 100644 --- a/chasm/lib/stream/handler.go +++ b/chasm/lib/stream/handler.go @@ -414,7 +414,7 @@ const longPollBuffer = time.Second // ReadRecords is a frontend read. It waits up to create_wait for the create and up to wait for // a Record, and answers an empty page after the wait. A page of an open Workflow run, run, keeps -// the run's floor on a Workflow Stream first, so no trim drops what it reads. +// the run's floor first, so no trim drops what it reads. func (h *handler) ReadRecords( ctx context.Context, req *streamservice.ReadRecordsRequest, @@ -487,7 +487,7 @@ func (h *handler) floorRun( from int64, run *ReaderRun, ) (*ReaderRun, error) { - if run == nil || !t.nested || req.GetWakeOnly() { + if run == nil || req.GetWakeOnly() { return nil, nil } lowers, err := read(ctx, t, func(s *Stream, cctx chasm.Context) (bool, error) { diff --git a/chasm/lib/stream/host.go b/chasm/lib/stream/host.go index b8ec5b1b14c..fbf774e7f1a 100644 --- a/chasm/lib/stream/host.go +++ b/chasm/lib/stream/host.go @@ -129,7 +129,7 @@ func targetOf(namespaceID, streamID string, ref *streamservice.StreamRef) (targe }, nil } -// newNestedStream makes a nested Stream, whose Workflow readers keep their floors. +// newNestedStream makes a nested Stream, whose Retention hides no Record that it still stores. func newNestedStream(ctx chasm.MutableContext) (*Stream, error) { s := NewStream(ctx, streampb.STREAM_PHASE_PENDING) s.Nested = true diff --git a/chasm/lib/stream/readers.go b/chasm/lib/stream/readers.go index 74ae8583e25..a44794f7c41 100644 --- a/chasm/lib/stream/readers.go +++ b/chasm/lib/stream/readers.go @@ -36,12 +36,10 @@ func readerRunOf(links []nexus.Link) *ReaderRun { return nil } -// readerFloor is the floor that a read from from sets for the run. It is false when the Stream -// keeps no floors, when the run's floor is already that low, or when the read fails anyway. +// readerFloor is the floor that a read from from sets for the run. It is false when the run's +// floor is already that low, or when the read fails anyway. A client's Stream keeps floors too, +// since the size cap would otherwise drop what a Workflow's replay reads again. func (s *Stream) readerFloor(ctx chasm.Context, runID string, from int64) (int64, bool) { - if !s.Nested { - return 0, false - } if from == 0 { from = s.TrimmedOffset + 1 } diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index 3eebfc0c164..387de5f068b 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -610,11 +610,13 @@ func (s *Stream) floor() (int64, bool) { return s.Floor, s.Floor > 0 && len(s.Readers) > 0 } -// retainedFrom is the offset below which the Retention hides the Records at now. A Workflow Stream -// hides none that it still stores, so a read from the start begins at the same offset on replay. +// retainedFrom is the offset below which the Retention hides the Records at now. A nested Stream, +// or one with Workflow readers, hides none that it still stores, so a read from the start begins +// at the same offset on replay. func (s *Stream) retainedFrom(now time.Time) int64 { retained := s.TrimmedOffset + 1 - if retention := s.GetRetention().AsDuration(); retention > 0 && !s.Nested { + retention := s.GetRetention().AsDuration() + if retention > 0 && !s.Nested && len(s.Readers) == 0 { if cutMs := now.Add(-retention).UnixMilli(); cutMs > 0 { retained = max(retained, cutMs< Date: Sun, 25 Oct 2026 01:12:57 -0700 Subject: [PATCH 11/23] Dropped the waiting reads of closed reader runs at the readers check. A wake outlived its run until the next write or its 7-day wait, so dead wakes filled stream.maxParkedReads. --- chasm/lib/stream/readers.go | 4 ++-- chasm/lib/stream/state.go | 9 ++++++++- chasm/lib/stream/stream_test.go | 34 +++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/chasm/lib/stream/readers.go b/chasm/lib/stream/readers.go index a44794f7c41..71df5924bdb 100644 --- a/chasm/lib/stream/readers.go +++ b/chasm/lib/stream/readers.go @@ -111,8 +111,8 @@ func (s *Stream) armReadersCheck(ctx chasm.MutableContext, limits Limits) { &streampb.StreamReadersCheckTask{Deadline: s.ReadersCheckTime}) } -// readersCheckTaskHandler drops the floors of the reader runs that closed. A run that continued -// as new is closed too, since the new run's History has none of its reads. +// readersCheckTaskHandler drops the floors and the waiting reads of the reader runs that closed. +// A run that continued as new is closed too, since the new run's History has none of its reads. type readersCheckTaskHandler struct { chasm.SideEffectTaskHandlerBase[*streampb.StreamReadersCheckTask] diff --git a/chasm/lib/stream/state.go b/chasm/lib/stream/state.go index e237ae3d3a2..4687423c14a 100644 --- a/chasm/lib/stream/state.go +++ b/chasm/lib/stream/state.go @@ -1,6 +1,7 @@ package stream import ( + "slices" "strings" "go.temporal.io/api/streamservice/v1" @@ -84,11 +85,17 @@ func (s *Stream) setReader(ctx chasm.MutableContext, run ReaderRun, offset int64 } } -// dropReaders drops the floors of runs that closed, and finds the lowest floor of the rest. +// dropReaders drops the floors and the waiting reads of runs that closed, and finds the lowest +// floor of the rest. A closed run takes no answer, and its wake would hold a slot for days. func (s *Stream) dropReaders(ctx chasm.MutableContext, runIDs []string) { for _, runID := range runIDs { delete(s.Readers, runID) } + if len(runIDs) > 0 { + s.resolveReads(ctx, func(read *ParkedRead) (readOutcome, bool) { + return readOutcome{err: errCanceled}, slices.Contains(runIDs, read.GetReaderRunId()) + }) + } s.Floor = 0 for runID := range s.Readers { floor := s.reader(ctx, runID) diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index e262b7f7797..48ae89d6ed9 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -717,6 +717,40 @@ func TestReaderFloorKeepsItsRecords(t *testing.T) { require.ErrorContains(t, err, ReasonExpired) } +// The readers check drops the wake of a run that closed, so dead wakes don't fill the cap of +// waiting reads until their wait ends. +func TestReadersCheckDropsTheWakeOfAClosedRun(t *testing.T) { + h := newHarness(t) + _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) + require.NoError(t, err) + result, err := chasm.UpdateWithStartExecution(h.ctx, h.key, StartForRead, (*Stream).ReadOrPark, + ReadInput{ + Request: &streamservice.ReadRecordsRequest{WakeOnly: true}, + RequestID: "wake", + Callback: &commonpb.Callback{Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{Url: chasm.NexusCompletionHandlerURL}, + }}, + Reader: &ReaderRun{WorkflowID: "wf", RunID: "run-1"}, + Limits: h.limits, + }, + chasm.WithBusinessIDPolicy( + chasm.BusinessIDReusePolicyRejectDuplicate, chasm.BusinessIDConflictPolicyUseExisting), + ) + require.NoError(t, err) + require.Nil(t, result.UpdateOutput) + require.False(t, h.parked("wake").GetResolved()) + + _, _, err = chasm.UpdateComponent(h.ctx, h.ref(), + func(s *Stream, mctx chasm.MutableContext, _ struct{}) (struct{}, error) { + s.dropReaders(mctx, []string{"run-1"}) + return struct{}{}, nil + }, struct{}{}) + require.NoError(t, err) + read := h.parked("wake") + require.True(t, read.GetResolved()) + require.NotNil(t, read.GetFailure().GetCanceledFailureInfo()) +} + // A client's Stream keeps the floor of a Workflow reader too, so the size cap can't drop what the // reader's replay reads again. The append that would drop it fails instead. func TestReaderFloorKeepsTheRecordsOfAClientStream(t *testing.T) { From 13e8d74fbbb74b3a23da26cb5ef7bf4cc19623b5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:14:39 -0700 Subject: [PATCH 12/23] Kept the chain's Streams open when a reset terminates a later run. The reset run carries the chain on, but the terminated run counted as the chain's end and closed the first run's Streams. --- .../history/workflow/mutable_state_impl.go | 5 +- tests/stream_test.go | 48 +++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index a0aa8ed7ff6..305691bd742 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -7528,8 +7528,9 @@ func (ms *MutableStateImpl) processCloseCallbacksChasm(closeEvent *historypb.His } firstRunID := ms.GetExecutionInfo().GetFirstExecutionRunId() - // A cron run's end isn't the chain's end. The next run writes on to the chain's Streams. - chainEnds := ms.executionInfo.NewExecutionRunId == "" + // A cron run's end isn't the chain's end, nor is the end of a run that a reset terminates. The + // next run or the reset run writes on to the chain's Streams. + chainEnds := ms.executionInfo.NewExecutionRunId == "" && !ms.executionInfo.GetWorkflowWasReset() hasStreams := chainEnds && wf.HasChainStreams(readCtx, firstRunID) // Return early if there are no chasm callbacks to process. diff --git a/tests/stream_test.go b/tests/stream_test.go index e98d206885f..d10071d857b 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -839,6 +839,54 @@ func (s *StreamTestSuite) TestStreamClosesWithTheLastRunOfTheChain() { s.Equal(streamservice.CLOSE_STATUS_COMPLETED, closed.GetCloseStatus()) } +// A reset to an earlier run of a continued-as-new chain terminates the current run, but the +// chain goes on in the reset run, so its Streams stay open. +func (s *StreamTestSuite) TestResetToAnEarlierRunKeepsTheStreamOpen() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + ns := env.Namespace().String() + owned := s.closeWithRun(sc, env, c) + first := c.run.GetRunID() + c.respond(&commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ + ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb. + ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, + TaskQueue: &taskqueuepb.TaskQueue{ + Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + }, + }, + }, + }) + s.NotEqual(first, c.current) + + reset, err := env.FrontendClient().ResetWorkflowExecution(s.Context(), + &workflowservice.ResetWorkflowExecutionRequest{ + Namespace: ns, + WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: c.run.GetID(), RunId: first}, + Reason: "test", + WorkflowTaskFinishEventId: 4, + RequestId: uuid.NewString(), + }) + s.NoError(err) + s.T().Cleanup(func() { + _ = env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), reset.GetRunId(), "test cleanup") + }) + + // The chain's end would close the Stream through a task on the terminated run. + time.Sleep(2 * time.Second) //nolint:forbidigo // The close task runs on real time. + described, err := sc.DescribeStream(s.Context(), + &streamservice.DescribeStreamRequest{Namespace: ns, Stream: owned}) + s.NoError(err) + s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) + + // The reset run's end is the chain's end. + s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), reset.GetRunId(), "stop")) + s.Equal(streamservice.CLOSE_STATUS_TERMINATED, s.readClose(sc, env, owned).GetCloseStatus()) +} + // directRead reads through the direct System Nexus call, which adds no History event. // A link names the reading Workflow run. func (s *StreamTestSuite) directRead( From 336a826eb3331ad1f9e468464cd5527f84d84266 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:19:14 -0700 Subject: [PATCH 13/23] Held a first run with Streams with the passive policy on a standby. The active policy fails with NamespaceNotActive on a standby, and the queue retried the delete task with no end. --- service/history/timer_queue_task_executor_base.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/service/history/timer_queue_task_executor_base.go b/service/history/timer_queue_task_executor_base.go index 01e1608c5b7..e7c7635eaac 100644 --- a/service/history/timer_queue_task_executor_base.go +++ b/service/history/timer_queue_task_executor_base.go @@ -152,7 +152,12 @@ func (t *timerQueueTaskExecutorBase) executeDeleteHistoryEventTask( BranchToken: task.BranchToken, ArchetypeID: task.ArchetypeID, }) - return weContext.SubmitClosedWorkflowSnapshot(ctx, t.shardContext, historyi.TransactionPolicyActive) + // A standby cluster writes the closed run as its replication does. + policy := historyi.TransactionPolicyActive + if !t.isActive { + policy = historyi.TransactionPolicyPassive + } + return weContext.SubmitClosedWorkflowSnapshot(ctx, t.shardContext, policy) } } From f89f6dd2d99904c01c4cae082e4ab2ac70772e41 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:19:34 -0700 Subject: [PATCH 14/23] Held a closed Standalone Activity until its Streams expire. The namespace retention deleted the Activity and its Streams, though a closed Stream stays readable for its Retention. A closed host runs no Stream task, so the hold reads each Stream's expiry time. --- chasm/lib/activity/stream.go | 14 ++++ chasm/lib/stream/host.go | 10 +++ .../history/timer_queue_task_executor_base.go | 73 +++++++++++++------ tests/stream_test.go | 63 ++++++++++++++++ 4 files changed, 139 insertions(+), 21 deletions(-) diff --git a/chasm/lib/activity/stream.go b/chasm/lib/activity/stream.go index 6634b859960..d38f166fbb3 100644 --- a/chasm/lib/activity/stream.go +++ b/chasm/lib/activity/stream.go @@ -81,6 +81,20 @@ func (a *Activity) checkAttempt( return nil } +// HoldsStreams says whether a Stream of the closed Activity is still readable, so the Activity +// outlives the namespace retention that would delete it. Each Stream closed with the Activity. +func (a *Activity) HoldsStreams(ctx chasm.Context) bool { + now := ctx.Now(a) + for _, field := range a.Streams { + if s, ok := field.TryGet(ctx); ok { + if until, held := s.HeldUntil(); held && until.After(now) { + return true + } + } + } + return false +} + // endStreams closes the Activity's Streams with its end. Each terminal transition passes its // status, since the transition sets the Activity's status after its apply function. func (a *Activity) endStreams(ctx chasm.MutableContext, status activitypb.ActivityExecutionStatus) { diff --git a/chasm/lib/stream/host.go b/chasm/lib/stream/host.go index fbf774e7f1a..293c1b768b4 100644 --- a/chasm/lib/stream/host.go +++ b/chasm/lib/stream/host.go @@ -3,6 +3,7 @@ package stream import ( "context" "errors" + "time" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -64,6 +65,15 @@ func (s *Stream) EndChain(ctx chasm.MutableContext, end RunEnd, limits Limits) { s.finish(ctx, limits) } +// HeldUntil is when a closed Stream expires, false for a Stream that isn't closed. A closed host +// runs no task of its Streams, so the close time and the Retention tell when the Stream expires. +func (s *Stream) HeldUntil() (time.Time, bool) { + if s.Phase != streampb.STREAM_PHASE_CLOSED { + return time.Time{}, false + } + return s.closedExpiryAt(), true +} + // EndOwner closes a Standalone Activity's Stream with the Activity's end, in the transition that // ends it. A retry doesn't end the Activity, so the Stream goes on across attempts. func (s *Stream) EndOwner(ctx chasm.MutableContext, end RunEnd, limits Limits) { diff --git a/service/history/timer_queue_task_executor_base.go b/service/history/timer_queue_task_executor_base.go index e7c7635eaac..da8b42c7017 100644 --- a/service/history/timer_queue_task_executor_base.go +++ b/service/history/timer_queue_task_executor_base.go @@ -11,6 +11,7 @@ import ( enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" + chasmstream "go.temporal.io/server/chasm/lib/stream" "go.temporal.io/server/common/locks" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" @@ -137,28 +138,28 @@ func (t *timerQueueTaskExecutorBase) executeDeleteHistoryEventTask( return err } - if task.ArchetypeID == chasm.WorkflowArchetypeID && mutableState.ChasmEnabled() { - wf, chasmCtx, err := mutableState.ChasmWorkflowComponentReadOnly(ctx) - if err != nil { - return err - } - // The first run of a chain keeps the chain's Streams, which the chain may still write - // after this run closed. The run stays one more retention, until its Streams expire. - if wf.HoldsStreams(chasmCtx) { - mutableState.AddTasks(&tasks.DeleteHistoryEventTask{ - WorkflowKey: task.WorkflowKey, - VisibilityTimestamp: t.shardContext.GetTimeSource().Now().Add(mutableState.GetNamespaceEntry().Retention()), - Version: task.Version, - BranchToken: task.BranchToken, - ArchetypeID: task.ArchetypeID, - }) - // A standby cluster writes the closed run as its replication does. - policy := historyi.TransactionPolicyActive - if !t.isActive { - policy = historyi.TransactionPolicyPassive - } - return weContext.SubmitClosedWorkflowSnapshot(ctx, t.shardContext, policy) + holds, err := holdsStreams(ctx, task.ArchetypeID, mutableState) + if err != nil { + return err + } + // The first run of a chain keeps the chain's Streams, which the chain may still write after + // this run closed. A Standalone Activity keeps its Streams readable for their Retention. The + // execution stays one more retention, until its Streams expire. + if holds { + mutableState.AddTasks(&tasks.DeleteHistoryEventTask{ + WorkflowKey: task.WorkflowKey, + VisibilityTimestamp: t.shardContext.GetTimeSource().Now(). + Add(mutableState.GetNamespaceEntry().Retention()), + Version: task.Version, + BranchToken: task.BranchToken, + ArchetypeID: task.ArchetypeID, + }) + // A standby cluster writes the closed execution as its replication does. + policy := historyi.TransactionPolicyActive + if !t.isActive { + policy = historyi.TransactionPolicyPassive } + return weContext.SubmitClosedWorkflowSnapshot(ctx, t.shardContext, policy) } return t.deleteManager.DeleteWorkflowExecutionByRetention( @@ -171,6 +172,36 @@ func (t *timerQueueTaskExecutorBase) executeDeleteHistoryEventTask( ) } +// streamHost is the root component of an execution that nests Streams. +type streamHost interface { + HoldsStreams(chasm.Context) bool +} + +// holdsStreams says whether a closed execution nests a Stream that is still readable. Only a +// Workflow and a Standalone Activity nest Streams. +func holdsStreams( + ctx context.Context, + archetypeID chasm.ArchetypeID, + mutableState historyi.MutableState, +) (bool, error) { + nests := archetypeID == chasm.WorkflowArchetypeID || + archetypeID == chasmstream.ActivityArchetypeID + if !nests || !mutableState.ChasmEnabled() { + return false, nil + } + tree, ok := mutableState.ChasmTree().(*chasm.Node) + if !ok { + return false, nil + } + chasmCtx := chasm.NewContext(ctx, tree) + root, err := tree.ComponentByPath(chasmCtx, nil) + if err != nil { + return false, err + } + host, ok := root.(streamHost) + return ok && host.HoldsStreams(chasmCtx), nil +} + func (t *timerQueueTaskExecutorBase) deleteHistoryBranch( ctx context.Context, branchToken []byte, diff --git a/tests/stream_test.go b/tests/stream_test.go index d10071d857b..5e09d5815d5 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -1012,6 +1012,69 @@ func (s *StreamTestSuite) TestChainOutlivesTheRetentionOfItsFirstRun() { s.Equal([]string{"late"}, kindsAndTopics(page)) } +// A closed Standalone Activity keeps its Streams readable for their Retention, so the namespace +// retention can't delete them with the Activity. +func (s *StreamTestSuite) TestActivityStreamOutlivesTheRetentionOfItsActivity() { + env := testcore.NewEnv(s.T(), + testcore.WithDedicatedCluster(), + testcore.WithDynamicConfig(stream.Enabled, true), + testcore.WithDynamicConfig(dynamicconfig.NamespaceMinRetentionGlobal, time.Duration(0)), + testcore.WithDynamicConfig(dynamicconfig.NamespaceMinRetentionLocal, time.Duration(0)), + testcore.WithDynamicConfig(dynamicconfig.RetentionTimerJitterDuration, time.Duration(0)), + ) + ctx := s.Context() + ns := env.Namespace().String() + _, err := env.FrontendClient().UpdateNamespace(ctx, &workflowservice.UpdateNamespaceRequest{ + Namespace: ns, + Config: &namespacepb.NamespaceConfig{WorkflowExecutionRetentionTtl: durationpb.New(time.Second)}, + }) + s.NoError(err) + // Wait for the namespace cache to see the new retention. + <-time.After(2 * testcore.NamespaceCacheRefreshInterval) + + activityID := "tool-" + uuid.NewString() + taskQueue := &taskqueuepb.TaskQueue{Name: "tools-" + uuid.NewString(), Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + started, err := env.FrontendClient().StartActivityExecution(ctx, + &workflowservice.StartActivityExecutionRequest{ + Namespace: ns, ActivityId: activityID, ActivityType: &commonpb.ActivityType{Name: "tool"}, + Identity: "test", TaskQueue: taskQueue, StartToCloseTimeout: durationpb.New(time.Minute), + }) + s.NoError(err) + task, err := env.FrontendClient().PollActivityTaskQueue(ctx, + &workflowservice.PollActivityTaskQueueRequest{Namespace: ns, TaskQueue: taskQueue, Identity: "test"}) + s.NoError(err) + id := testcore.RandomizeStr("tokens") + owned := &streamservice.StreamRef{Id: id, ActivityId: activityID, ActivityRunId: started.GetRunId()} + owner := &streamservice.Producer{ + Kind: streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY, + Id: "sa:" + activityID + ":" + started.GetRunId(), + Epoch: &streamservice.ProducerEpoch{ + AttemptScheduledTime: task.GetCurrentAttemptScheduledTime(), Attempt: task.GetAttempt(), + }, + } + sc := s.streamClient(env) + _, err = sc.CreateStream(ctx, &streamservice.CreateStreamRequest{ + Namespace: ns, StreamId: id, Ref: owned, Owner: owner, TaskToken: task.GetTaskToken(), + }) + s.NoError(err) + _, err = sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ + Namespace: ns, Stream: owned, Producer: owner, Sequence: 1, Records: records("tokens"), + TaskToken: task.GetTaskToken(), + }) + s.NoError(err) + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, + &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: ns, TaskToken: task.GetTaskToken(), Identity: "test", + }) + s.NoError(err) + // Several namespace retentions pass after the Activity's end. + <-time.After(4 * time.Second) + + page, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{Namespace: ns, Stream: owned}) + s.NoError(err) + s.Equal([]string{"tokens", "close"}, kindsAndTopics(page)) +} + // An append that names a Workflow Stream by its bare id, or a Stream that doesn't exist, fails. // It never writes somewhere a reader of the Workflow Stream can't see. func (s *StreamTestSuite) TestAppendToAStreamThatIsNotThereFails() { From f5bdac770a24d77a88854f89353a100425ba05db Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:21:24 -0700 Subject: [PATCH 15/23] Skipped the chain end's write to a first run with no Streams. A later run can't tell whether the first run has Streams, so every continue-as-new or retry chain end wrote the first run. --- chasm/lib/stream/host.go | 12 ++++++++++++ tests/stream_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/chasm/lib/stream/host.go b/chasm/lib/stream/host.go index 293c1b768b4..ed787222c93 100644 --- a/chasm/lib/stream/host.go +++ b/chasm/lib/stream/host.go @@ -302,6 +302,18 @@ func (h *closeChainTaskHandler) Execute( nested: true, host: chasm.WorkflowArchetypeID, } + // A later run can't tell whether the first run has Streams, so every chain end comes here. A + // read first spares the first run a write when it has none. + nests, err := chasm.ReadComponent(ctx, first.hostRef(), + func(h Host, _ chasm.Context, _ struct{}) (bool, error) { + return len(h.NestedStreams()) > 0, nil + }, struct{}{}) + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { + return nil + } + if err != nil || !nests { + return err + } _, _, err = chasm.UpdateComponent(ctx, first.hostRef(), func(h Host, mctx chasm.MutableContext, _ struct{}) (struct{}, error) { for _, field := range h.NestedStreams() { diff --git a/tests/stream_test.go b/tests/stream_test.go index 5e09d5815d5..3ae2e3e7f1a 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -16,6 +16,8 @@ import ( taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/sdk/client" + "go.temporal.io/server/api/adminservice/v1" + "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/stream" "go.temporal.io/server/common/dynamicconfig" commonnexus "go.temporal.io/server/common/nexus" @@ -887,6 +889,41 @@ func (s *StreamTestSuite) TestResetToAnEarlierRunKeepsTheStreamOpen() { s.Equal(streamservice.CLOSE_STATUS_TERMINATED, s.readClose(sc, env, owned).GetCloseStatus()) } +// A chain whose first run has no Stream ends with no write to that run. +func (s *StreamTestSuite) TestChainEndLeavesAFirstRunWithNoStreamAlone() { + env := s.newEnv() + c := s.newCaller(env) + ns := env.Namespace().String() + first := c.run.GetRunID() + c.respond(&commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ + ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb. + ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, + TaskQueue: &taskqueuepb.TaskQueue{ + Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + }, + }, + }, + }) + transitions := func() int64 { + resp, err := env.AdminClient().DescribeMutableState(s.Context(), + &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: c.run.GetID(), RunId: first}, + Archetype: chasm.WorkflowArchetype, + }) + s.NoError(err) + return resp.GetDatabaseMutableState().GetExecutionInfo().GetStateTransitionCount() + } + before := transitions() + + s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), c.current, "stop")) + time.Sleep(2 * time.Second) //nolint:forbidigo // The chain's close task runs on real time. + s.Equal(before, transitions()) +} + // directRead reads through the direct System Nexus call, which adds no History event. // A link names the reading Workflow run. func (s *StreamTestSuite) directRead( From c8ddf365028265550ee1fb79f178cef66a325f5f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:22:30 -0700 Subject: [PATCH 16/23] Ran the server's withdrawal rule and CompareEpochs on the crate vectors. The server has its own copy of both rules, and nothing checked it against the crate's. --- chasm/lib/stream/testdata/epochs.json | 55 + chasm/lib/stream/testdata/withdrawals.json | 1354 ++++++++++++++++++++ chasm/lib/stream/vectors_test.go | 158 +++ 3 files changed, 1567 insertions(+) create mode 100644 chasm/lib/stream/testdata/epochs.json create mode 100644 chasm/lib/stream/testdata/withdrawals.json create mode 100644 chasm/lib/stream/vectors_test.go diff --git a/chasm/lib/stream/testdata/epochs.json b/chasm/lib/stream/testdata/epochs.json new file mode 100644 index 00000000000..ce1fe07355c --- /dev/null +++ b/chasm/lib/stream/testdata/epochs.json @@ -0,0 +1,55 @@ +[ + { + "kind": "workflow", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 2, + "task_started_time": [ + 1760000000, + 5 + ] + }, + "key": "0000000000000008.00000002.0000000068e77800.00000005.0000000000000000.00000000.00000000", + "task_key": "0000000000000008.00000002.0000000068e77800.00000005" + }, + { + "kind": "local_activity", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 1760000000, + 5 + ], + "attempt": 3 + }, + "key": "0000000000000008.00000001.0000000068e77800.00000005.0000000000000000.00000000.00000003", + "task_key": "0000000000000008.00000001.0000000068e77800.00000005" + }, + { + "kind": "activity", + "epoch": { + "attempt_scheduled_time": [ + 1760000001, + 999999999 + ], + "attempt": 2 + }, + "key": "0000000000000000.00000000.0000000000000000.00000000.0000000068e77801.3b9ac9ff.00000002", + "task_key": "0000000000000000.00000000.0000000000000000.00000000" + }, + { + "kind": "client", + "epoch": { + "attempt": 1 + }, + "key": "0000000000000000.00000000.0000000000000000.00000000.0000000000000000.00000000.00000001", + "task_key": "0000000000000000.00000000.0000000000000000.00000000" + }, + { + "kind": "empty", + "epoch": {}, + "key": "0000000000000000.00000000.0000000000000000.00000000.0000000000000000.00000000.00000000", + "task_key": "0000000000000000.00000000.0000000000000000.00000000" + } +] diff --git a/chasm/lib/stream/testdata/withdrawals.json b/chasm/lib/stream/testdata/withdrawals.json new file mode 100644 index 00000000000..128c21425f1 --- /dev/null +++ b/chasm/lib/stream/testdata/withdrawals.json @@ -0,0 +1,1354 @@ +[ + { + "name": "e1 B at the failed task 5 goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e1 a Record at the completed task 8 stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e1 a Record before c stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e2 B goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 2, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e2 an older completed Record stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 2, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 2, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e3 D at a lower attempt of t goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 3, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 5, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e3 a Record of t itself stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 3, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 5, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 3, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e4 U at an earlier started time goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 9, + "task_attempt": 1, + "task_started_time": [ + 200, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 6, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 9, + "task_attempt": 1, + "task_started_time": [ + 150, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e5 D between c and t goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 5, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e5 a Record at c stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 5, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "c attempt 0, an earlier attempt at c goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 0, + "started_time": [ + 300, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 200, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "c attempt 0, c itself stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 0, + "started_time": [ + 300, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 2, + "task_started_time": [ + 300, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e1 a Local Activity of the failed task goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 2 + } + } + }, + "withdraws": true + }, + { + "name": "another run's Record stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r9", + "run_id": "r9", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "e6 r1's Record goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "withdraw_run": "r1" + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 20, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "e6 r1's Activity Record goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "withdraw_run": "r1" + }, + "record": { + "producer": { + "kind": "activity", + "id": "act:r1:a:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "e6 r2's own earlier Record of t stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [], + "withdraw_run": "r1" + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "an Activity retry withdraws attempt 1", + "supersession": { + "producer_kind": "activity", + "producer_id": "act:r1:a:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 130, + 0 + ], + "attempt": 2 + } + }, + "record": { + "producer": { + "kind": "activity", + "id": "act:r1:a:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "an Activity retry keeps another Activity", + "supersession": { + "producer_kind": "activity", + "producer_id": "act:r1:a:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 130, + 0 + ], + "attempt": 2 + } + }, + "record": { + "producer": { + "kind": "activity", + "id": "act:r1:b:s", + "run_id": "r1", + "epoch": { + "attempt_scheduled_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a client's newer epoch keeps its earlier Records", + "supersession": { + "producer_kind": "client", + "producer_id": "client:ingest", + "epoch": { + "attempt": 2 + } + }, + "record": { + "producer": { + "kind": "client", + "id": "client:ingest", + "epoch": { + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "no epoch, r1's Record goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "withdraw_run": "r1", + "failed_tasks": [ + 5 + ] + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": true + }, + { + "name": "no epoch, r1's Local Activity Record goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "withdraw_run": "r1", + "failed_tasks": [ + 5 + ] + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "no epoch, r2's Local Activity Record at a failed task stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "withdraw_run": "r1", + "failed_tasks": [ + 5 + ] + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r2:k", + "run_id": "r2", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "no epoch, r2's Record stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r2", + "run_id": "r2", + "withdraw_run": "r1", + "failed_tasks": [ + 5 + ] + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r2", + "run_id": "r2", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity's newer epoch withdraws a lower attempt", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 2 + } + } + }, + "withdraws": true + }, + { + "name": "a Local Activity's newer epoch withdraws an earlier task", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 4 + } + } + }, + "withdraws": true + }, + { + "name": "a Local Activity's newer epoch keeps its own epoch", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity's newer epoch keeps another call", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:j", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity's item keeps the run's Workflow Record", + "supersession": { + "producer_kind": "local_activity", + "producer_id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 3 + } + }, + "record": { + "producer": { + "kind": "workflow", + "id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 5, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ] + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity before c stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 3, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity at c stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity between c and t goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "a Local Activity of a lower attempt of t goes", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": true + }, + { + "name": "a Local Activity of t itself stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity of another run stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r9:k", + "run_id": "r9", + "epoch": { + "task_started_event_id": 11, + "task_attempt": 1, + "task_started_time": [ + 100, + 0 + ], + "attempt": 1 + } + } + }, + "withdraws": false + }, + { + "name": "a Local Activity Record with no epoch stays", + "supersession": { + "producer_kind": "workflow", + "producer_id": "wf:r1", + "run_id": "r1", + "epoch": { + "task_started_event_id": 14, + "task_attempt": 2, + "task_started_time": [ + 100, + 0 + ] + }, + "failed_tasks": [ + 5 + ], + "previous_completed_task": { + "started_event_id": 8, + "attempt": 1, + "started_time": [ + 100, + 0 + ] + } + }, + "record": { + "producer": { + "kind": "local_activity", + "id": "la:r1:k", + "run_id": "r1" + } + }, + "withdraws": false + } +] diff --git a/chasm/lib/stream/vectors_test.go b/chasm/lib/stream/vectors_test.go new file mode 100644 index 00000000000..81b26f4a541 --- /dev/null +++ b/chasm/lib/stream/vectors_test.go @@ -0,0 +1,158 @@ +package stream + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/streamservice/v1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// The files in testdata are copies of the crate's shared vectors, at moetemp/sdk-streams +// crates/streams/testdata, so the server's Producer rules and the Stores agree. + +type vectorTime [2]int64 + +func (t *vectorTime) proto() *timestamppb.Timestamp { + if t == nil { + return nil + } + return ×tamppb.Timestamp{Seconds: t[0], Nanos: int32(t[1])} +} + +type vectorEpoch struct { + TaskStartedEventID int64 `json:"task_started_event_id"` + TaskAttempt int32 `json:"task_attempt"` + TaskStartedTime *vectorTime `json:"task_started_time"` + AttemptScheduledTime *vectorTime `json:"attempt_scheduled_time"` + Attempt int32 `json:"attempt"` +} + +func (e *vectorEpoch) proto() *streamservice.ProducerEpoch { + if e == nil { + return nil + } + return &streamservice.ProducerEpoch{ + TaskStartedEventId: e.TaskStartedEventID, + TaskAttempt: e.TaskAttempt, + TaskStartedTime: e.TaskStartedTime.proto(), + AttemptScheduledTime: e.AttemptScheduledTime.proto(), + Attempt: e.Attempt, + } +} + +var vectorKinds = map[string]streamservice.ProducerKind{ + "workflow": streamservice.PRODUCER_KIND_WORKFLOW, + "local_activity": streamservice.PRODUCER_KIND_LOCAL_ACTIVITY, + "activity": streamservice.PRODUCER_KIND_ACTIVITY, + "standalone_activity": streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY, + "client": streamservice.PRODUCER_KIND_CLIENT, +} + +func readVectors(t *testing.T, name string, into any) { + data, err := os.ReadFile("testdata/" + name) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(data, into)) +} + +// vectorWithdraws applies a Superseded Record to a Record with the server's rules: the task rule +// of withdrawTasks for a Workflow item, and CompareEpochs for the other kinds that withdraw. +func vectorWithdraws(s *streamservice.Supersession, p *streamservice.Producer) bool { + if s.GetWithdrawRun() != "" && p.GetRunId() == s.GetWithdrawRun() { + return true + } + if s.GetEpoch() == nil || p.GetEpoch() == nil { + return false + } + if s.GetProducerKind() != streamservice.PRODUCER_KIND_WORKFLOW { + return withdrawsEpochs(s.GetProducerKind()) && p.GetKind() == s.GetProducerKind() && + p.GetId() == s.GetProducerId() && CompareEpochs(p.GetEpoch(), s.GetEpoch()) < 0 + } + switch p.GetKind() { + case streamservice.PRODUCER_KIND_WORKFLOW: + if p.GetId() != s.GetProducerId() { + return false + } + case streamservice.PRODUCER_KIND_LOCAL_ACTIVITY: + if p.GetRunId() != s.GetRunId() { + return false + } + default: + return false + } + index := []*streamservice.WorkflowTaskEpoch{taskOf(p.GetEpoch())} + split := withdrawTasks(index, taskOf(s.GetEpoch()), s.GetPreviousCompletedTask(), + s.GetFailedTasks()) + return len(split.withdrawn) > 0 +} + +func TestWithdrawalsMatchTheCrateVectors(t *testing.T) { + var vectors []struct { + Name string `json:"name"` + Supersession struct { + ProducerKind string `json:"producer_kind"` + ProducerID string `json:"producer_id"` + RunID string `json:"run_id"` + Epoch *vectorEpoch `json:"epoch"` + FailedTasks []int64 `json:"failed_tasks"` + WithdrawRun string `json:"withdraw_run"` + Completed *struct { + StartedEventID int64 `json:"started_event_id"` + Attempt int32 `json:"attempt"` + StartedTime *vectorTime `json:"started_time"` + } `json:"previous_completed_task"` + } `json:"supersession"` + Record struct { + Producer struct { + Kind string `json:"kind"` + ID string `json:"id"` + RunID string `json:"run_id"` + Epoch *vectorEpoch `json:"epoch"` + } `json:"producer"` + } `json:"record"` + Withdraws bool `json:"withdraws"` + } + readVectors(t, "withdrawals.json", &vectors) + require.NotEmpty(t, vectors) + for _, v := range vectors { + s := &streamservice.Supersession{ + ProducerKind: vectorKinds[v.Supersession.ProducerKind], + ProducerId: v.Supersession.ProducerID, + RunId: v.Supersession.RunID, + Epoch: v.Supersession.Epoch.proto(), + FailedTasks: v.Supersession.FailedTasks, + WithdrawRun: v.Supersession.WithdrawRun, + } + if c := v.Supersession.Completed; c != nil { + s.PreviousCompletedTask = &streamservice.WorkflowTaskEpoch{ + StartedEventId: c.StartedEventID, + Attempt: c.Attempt, + StartedTime: c.StartedTime.proto(), + } + } + p := v.Record.Producer + producer := &streamservice.Producer{ + Kind: vectorKinds[p.Kind], Id: p.ID, RunId: p.RunID, Epoch: p.Epoch.proto(), + } + require.Equal(t, v.Withdraws, vectorWithdraws(s, producer), v.Name) + } +} + +// The crate's epoch keys sort as its epochs compare, so CompareEpochs must give the same order. +func TestCompareEpochsMatchesTheCrateKeys(t *testing.T) { + var vectors []struct { + Epoch vectorEpoch `json:"epoch"` + Key string `json:"key"` + } + readVectors(t, "epochs.json", &vectors) + require.NotEmpty(t, vectors) + for _, a := range vectors { + for _, b := range vectors { + require.Equal(t, strings.Compare(a.Key, b.Key), CompareEpochs(a.Epoch.proto(), b.Epoch.proto()), + "%s against %s", a.Key, b.Key) + } + } +} From e4a265e9a2c58940b8a2436cae1609bce35dad29 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:28:42 -0700 Subject: [PATCH 17/23] Wrapped our Go lines at 100 columns. The stack already wrapped the chain-end code at 100. Two rows of upstream tables keep their gofmt alignment. --- chasm/lib/stream/errors.go | 3 +- chasm/lib/stream/frontend.go | 45 +- chasm/lib/stream/grpc.go | 25 +- chasm/lib/stream/handler.go | 101 ++++- chasm/lib/stream/host.go | 45 +- chasm/lib/stream/library.go | 9 +- chasm/lib/stream/nexus.go | 99 +++-- chasm/lib/stream/nexus_test.go | 72 +++- chasm/lib/stream/parked_read.go | 33 +- chasm/lib/stream/producer.go | 38 +- chasm/lib/stream/read.go | 42 +- chasm/lib/stream/readers.go | 34 +- chasm/lib/stream/stream.go | 90 +++- chasm/lib/stream/stream_test.go | 197 +++++++-- chasm/lib/stream/tasks.go | 3 +- chasm/lib/stream/vectors_test.go | 10 +- service/frontend/fx.go | 3 +- service/frontend/nexus_direct.go | 50 ++- tests/stream_test.go | 706 ++++++++++++++++++++----------- 19 files changed, 1156 insertions(+), 449 deletions(-) diff --git a/chasm/lib/stream/errors.go b/chasm/lib/stream/errors.go index 9f7b98aef49..b1ab06c883a 100644 --- a/chasm/lib/stream/errors.go +++ b/chasm/lib/stream/errors.go @@ -48,7 +48,8 @@ var reasons = []string{ // StreamErrorDetails in the status details. func NewError(reason string, format string, args ...any) error { st := status.New(reasonCode(reason), reason+": "+fmt.Sprintf(format, args...)) - if detailed, err := st.WithDetails(&streamservice.StreamErrorDetails{Reason: reason}); err == nil { + detailed, err := st.WithDetails(&streamservice.StreamErrorDetails{Reason: reason}) + if err == nil { st = detailed } return serviceerror.FromStatus(st) diff --git a/chasm/lib/stream/frontend.go b/chasm/lib/stream/frontend.go index 9dae42dda4e..958e0595066 100644 --- a/chasm/lib/stream/frontend.go +++ b/chasm/lib/stream/frontend.go @@ -33,11 +33,21 @@ func NewFrontendHandler( // mutable state, so a call served elsewhere would read a replica or write a copy. func RedirectableMethods() map[string]func() any { return map[string]func() any{ - streamservice.StreamService_CreateStream_FullMethodName: func() any { return &streamservice.CreateStreamResponse{} }, - streamservice.StreamService_AppendRecords_FullMethodName: func() any { return &streamservice.AppendRecordsResponse{} }, - streamservice.StreamService_CloseStream_FullMethodName: func() any { return &streamservice.CloseStreamResponse{} }, - streamservice.StreamService_ReadRecords_FullMethodName: func() any { return &streamservice.ReadRecordsResponse{} }, - streamservice.StreamService_DescribeStream_FullMethodName: func() any { return &streamservice.DescribeStreamResponse{} }, + streamservice.StreamService_CreateStream_FullMethodName: func() any { + return &streamservice.CreateStreamResponse{} + }, + streamservice.StreamService_AppendRecords_FullMethodName: func() any { + return &streamservice.AppendRecordsResponse{} + }, + streamservice.StreamService_CloseStream_FullMethodName: func() any { + return &streamservice.CloseStreamResponse{} + }, + streamservice.StreamService_ReadRecords_FullMethodName: func() any { + return &streamservice.ReadRecordsResponse{} + }, + streamservice.StreamService_DescribeStream_FullMethodName: func() any { + return &streamservice.DescribeStreamResponse{} + }, } } @@ -70,7 +80,10 @@ func (h *FrontendHandler) CreateStream( if err != nil { return nil, err } - resp, err := h.client.CreateStream(ctx, &streampb.CreateStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.CreateStream( + ctx, + &streampb.CreateStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } @@ -82,7 +95,10 @@ func (h *FrontendHandler) AppendRecords( if err != nil { return nil, err } - resp, err := h.client.AppendRecords(ctx, &streampb.AppendRecordsRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.AppendRecords( + ctx, + &streampb.AppendRecordsRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } @@ -94,7 +110,10 @@ func (h *FrontendHandler) CloseStream( if err != nil { return nil, err } - resp, err := h.client.CloseStream(ctx, &streampb.CloseStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.CloseStream( + ctx, + &streampb.CloseStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } @@ -106,7 +125,10 @@ func (h *FrontendHandler) ReadRecords( if err != nil { return nil, err } - resp, err := h.client.ReadRecords(ctx, &streampb.ReadRecordsRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.ReadRecords( + ctx, + &streampb.ReadRecordsRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } @@ -118,6 +140,9 @@ func (h *FrontendHandler) DescribeStream( if err != nil { return nil, err } - resp, err := h.client.DescribeStream(ctx, &streampb.DescribeStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}) + resp, err := h.client.DescribeStream( + ctx, + &streampb.DescribeStreamRequest{NamespaceId: namespaceID, FrontendRequest: req}, + ) return resp.GetFrontendResponse(), err } diff --git a/chasm/lib/stream/grpc.go b/chasm/lib/stream/grpc.go index 230e38c69e1..b1124a4ca21 100644 --- a/chasm/lib/stream/grpc.go +++ b/chasm/lib/stream/grpc.go @@ -17,7 +17,10 @@ func newGRPCHandler(h *handler) *grpcHandler { return &grpcHandler{handler: h} } -func (g *grpcHandler) CreateStream(ctx context.Context, req *streampb.CreateStreamRequest) (*streampb.CreateStreamResponse, error) { +func (g *grpcHandler) CreateStream( + ctx context.Context, + req *streampb.CreateStreamRequest, +) (*streampb.CreateStreamResponse, error) { resp, err := g.handler.CreateStream(ctx, req.GetFrontendRequest()) if err != nil { return nil, err @@ -25,7 +28,10 @@ func (g *grpcHandler) CreateStream(ctx context.Context, req *streampb.CreateStre return &streampb.CreateStreamResponse{FrontendResponse: resp}, nil } -func (g *grpcHandler) AppendRecords(ctx context.Context, req *streampb.AppendRecordsRequest) (*streampb.AppendRecordsResponse, error) { +func (g *grpcHandler) AppendRecords( + ctx context.Context, + req *streampb.AppendRecordsRequest, +) (*streampb.AppendRecordsResponse, error) { resp, err := g.handler.AppendRecords(ctx, req.GetFrontendRequest()) if err != nil { return nil, err @@ -33,7 +39,10 @@ func (g *grpcHandler) AppendRecords(ctx context.Context, req *streampb.AppendRec return &streampb.AppendRecordsResponse{FrontendResponse: resp}, nil } -func (g *grpcHandler) CloseStream(ctx context.Context, req *streampb.CloseStreamRequest) (*streampb.CloseStreamResponse, error) { +func (g *grpcHandler) CloseStream( + ctx context.Context, + req *streampb.CloseStreamRequest, +) (*streampb.CloseStreamResponse, error) { resp, err := g.handler.CloseStream(ctx, req.GetFrontendRequest()) if err != nil { return nil, err @@ -41,7 +50,10 @@ func (g *grpcHandler) CloseStream(ctx context.Context, req *streampb.CloseStream return &streampb.CloseStreamResponse{FrontendResponse: resp}, nil } -func (g *grpcHandler) ReadRecords(ctx context.Context, req *streampb.ReadRecordsRequest) (*streampb.ReadRecordsResponse, error) { +func (g *grpcHandler) ReadRecords( + ctx context.Context, + req *streampb.ReadRecordsRequest, +) (*streampb.ReadRecordsResponse, error) { resp, err := g.handler.ReadRecords(ctx, req.GetFrontendRequest(), nil) if err != nil { return nil, err @@ -49,7 +61,10 @@ func (g *grpcHandler) ReadRecords(ctx context.Context, req *streampb.ReadRecords return &streampb.ReadRecordsResponse{FrontendResponse: resp}, nil } -func (g *grpcHandler) DescribeStream(ctx context.Context, req *streampb.DescribeStreamRequest) (*streampb.DescribeStreamResponse, error) { +func (g *grpcHandler) DescribeStream( + ctx context.Context, + req *streampb.DescribeStreamRequest, +) (*streampb.DescribeStreamResponse, error) { resp, err := g.handler.DescribeStream(ctx, req.GetFrontendRequest()) if err != nil { return nil, err diff --git a/chasm/lib/stream/handler.go b/chasm/lib/stream/handler.go index 10bf1931bc5..d11d38aacbc 100644 --- a/chasm/lib/stream/handler.go +++ b/chasm/lib/stream/handler.go @@ -24,8 +24,16 @@ type handler struct { historyClient resource.HistoryClient } -func newHandler(config *Config, namespaceRegistry namespace.Registry, historyClient resource.HistoryClient) *handler { - return &handler{config: config, namespaceRegistry: namespaceRegistry, historyClient: historyClient} +func newHandler( + config *Config, + namespaceRegistry namespace.Registry, + historyClient resource.HistoryClient, +) *handler { + return &handler{ + config: config, + namespaceRegistry: namespaceRegistry, + historyClient: historyClient, + } } // Reader is a Workflow read through System Nexus. @@ -49,7 +57,11 @@ func checkProducer(producer *streamservice.Producer) error { return invalid("The Producer has no kind") } if !strings.HasPrefix(producer.GetId(), prefix) { - return invalid("The Producer id %q doesn't start with %q, as its kind requires", producer.GetId(), prefix) + return invalid( + "The Producer id %q doesn't start with %q, as its kind requires", + producer.GetId(), + prefix, + ) } return nil } @@ -95,7 +107,10 @@ func CheckStreamID(id string) error { } // target resolves the namespace, the limits and the place of a Stream. -func (h *handler) target(namespaceName, streamID string, ref *streamservice.StreamRef) (target, Limits, error) { +func (h *handler) target( + namespaceName, streamID string, + ref *streamservice.StreamRef, +) (target, Limits, error) { if err := CheckStreamID(streamID); err != nil { return target{}, Limits{}, err } @@ -175,7 +190,8 @@ func (h *handler) CreateStream( } workflowOwner := req.GetOwner().GetKind() == streamservice.PRODUCER_KIND_WORKFLOW if req.GetCloseWithRun() && (!t.workflow() || !workflowOwner) { - return nil, invalid("close_with_run needs a Workflow owner and a ref with workflow_id and first_execution_run_id") + return nil, invalid("close_with_run needs a Workflow owner and a ref with workflow_id " + + "and first_execution_run_id") } if err := h.checkChain(ctx, t, req.GetOwner(), req.GetWithdrawRun()); err != nil { return nil, err @@ -186,8 +202,10 @@ func (h *handler) CreateStream( check := func(h Host, cctx chasm.Context) error { return h.CheckOwner(cctx, req.GetOwner(), req.GetTaskToken()) } - out, err = update(ctx, t, newNestedStream, check, - func(s *Stream, mctx chasm.MutableContext) (CreateOutput, error) { return s.Create(mctx, in) }) + create := func(s *Stream, mctx chasm.MutableContext) (CreateOutput, error) { + return s.Create(mctx, in) + } + out, err = update(ctx, t, newNestedStream, check, create) err = streamError(ctx, t, err) } else { out, err = h.createStandalone(ctx, t, in) @@ -205,7 +223,11 @@ func (h *handler) CreateStream( }, nil } -func (h *handler) createStandalone(ctx context.Context, t target, in CreateInput) (CreateOutput, error) { +func (h *handler) createStandalone( + ctx context.Context, + t target, + in CreateInput, +) (CreateOutput, error) { create := func(policy chasm.TransitionOption) (CreateOutput, error) { result, err := chasm.UpdateWithStartExecution(ctx, t.key, StartStream, func(s *Stream, mctx chasm.MutableContext, in CreateInput) (CreateOutput, error) { @@ -330,16 +352,24 @@ func (h *handler) DescribeStream( if err := h.applyDue(ctx, t, limits); err != nil { return nil, streamError(ctx, t, err) } - resp, err := read(ctx, t, func(s *Stream, _ chasm.Context) (*streamservice.DescribeStreamResponse, error) { - return s.Describe() - }) + resp, err := read( + ctx, + t, + func(s *Stream, _ chasm.Context) (*streamservice.DescribeStreamResponse, error) { + return s.Describe() + }, + ) return resp, streamError(ctx, t, err) } // applyDue runs a write step when a deadline such as the TTL passed, so that a read never sees // the Stream open after it expired. func (h *handler) applyDue(ctx context.Context, t target, limits Limits) error { - due, err := read(ctx, t, func(s *Stream, cctx chasm.Context) (bool, error) { return s.Due(cctx, limits) }) + due, err := read( + ctx, + t, + func(s *Stream, cctx chasm.Context) (bool, error) { return s.Due(cctx, limits) }, + ) if err != nil || !due { return err } @@ -364,7 +394,13 @@ func (h *handler) ReadOrPark( if err := checkLast(req, true); err != nil { return nil, err } - in := ReadInput{Request: req, RequestID: reader.RequestID, Callback: reader.Callback, Reader: reader.Run, Limits: limits} + in := ReadInput{ + Request: req, + RequestID: reader.RequestID, + Callback: reader.Callback, + Reader: reader.Run, + Limits: limits, + } if t.nested { resp, err := update(ctx, t, newNestedStream, nil, func(s *Stream, mctx chasm.MutableContext) (*streamservice.ReadRecordsResponse, error) { @@ -372,12 +408,19 @@ func (h *handler) ReadOrPark( }) return resp, streamError(ctx, t, err) } - readStandalone := func(policy chasm.TransitionOption) (*streamservice.ReadRecordsResponse, error) { - result, err := chasm.UpdateWithStartExecution(ctx, t.key, StartForRead, - func(s *Stream, mctx chasm.MutableContext, in ReadInput) (*streamservice.ReadRecordsResponse, error) { - resp, err := s.ReadOrPark(mctx, in) - return resp, carry(err) - }, in, policy) + readOrPark := func( + s *Stream, + mctx chasm.MutableContext, + in ReadInput, + ) (*streamservice.ReadRecordsResponse, error) { + resp, err := s.ReadOrPark(mctx, in) + return resp, carry(err) + } + readStandalone := func( + policy chasm.TransitionOption, + ) (*streamservice.ReadRecordsResponse, error) { + result, err := chasm.UpdateWithStartExecution(ctx, t.key, StartForRead, readOrPark, in, + policy) return result.UpdateOutput, uncarry(err) } resp, err := readStandalone(useExisting) @@ -394,7 +437,12 @@ func (h *handler) ReadOrPark( } // CancelRead completes a parked read as canceled. -func (h *handler) CancelRead(ctx context.Context, namespaceName string, ref *streamservice.StreamRef, requestID string) error { +func (h *handler) CancelRead( + ctx context.Context, + namespaceName string, + ref *streamservice.StreamRef, + requestID string, +) error { t, limits, err := h.target(namespaceName, ref.GetId(), ref) if err != nil { return err @@ -448,7 +496,11 @@ func (h *handler) ReadRecords( }) return resp, streamError(ctx, t, err) } - start, err := read(ctx, t, func(s *Stream, _ chasm.Context) (int64, error) { return s.StartOffset(req) }) + start, err := read( + ctx, + t, + func(s *Stream, _ chasm.Context) (int64, error) { return s.StartOffset(req) }, + ) if err != nil { return nil, streamError(ctx, t, err) } @@ -569,7 +621,12 @@ func (h *handler) waitForCreate(ctx context.Context, t target, until time.Time) // waitForChange waits until the Stream has an entry at or past cursor, or is no longer open, or // the deadline passes. The expiry is a timer task, so it ends the wait too. -func (h *handler) waitForChange(ctx context.Context, t target, cursor int64, deadline time.Time) error { +func (h *handler) waitForChange( + ctx context.Context, + t target, + cursor int64, + deadline time.Time, +) error { pollCtx, cancel := context.WithDeadline(ctx, deadline) defer cancel() cursor = max(cursor, 1) diff --git a/chasm/lib/stream/host.go b/chasm/lib/stream/host.go index ed787222c93..37eb7668272 100644 --- a/chasm/lib/stream/host.go +++ b/chasm/lib/stream/host.go @@ -122,7 +122,10 @@ func targetOf(namespaceID, streamID string, ref *streamservice.StreamRef) (targe }, nil } if ref.GetWorkflowId() == "" { - return target{key: chasm.ExecutionKey{NamespaceID: namespaceID, BusinessID: streamID}, id: streamID}, nil + return target{ + key: chasm.ExecutionKey{NamespaceID: namespaceID, BusinessID: streamID}, + id: streamID, + }, nil } if ref.GetFirstExecutionRunId() == "" { return target{}, invalid("A Workflow Stream ref needs first_execution_run_id") @@ -165,7 +168,9 @@ func update[O any]( } if !t.nested { out, _, err := chasm.UpdateComponent(ctx, chasm.NewComponentRef[*Stream](t.key), - func(s *Stream, mctx chasm.MutableContext, _ struct{}) (O, error) { return fn(s, mctx) }, struct{}{}) + func(s *Stream, mctx chasm.MutableContext, _ struct{}) (O, error) { + return fn(s, mctx) + }, struct{}{}) return out, uncarry(err) } out, _, err := chasm.UpdateComponent(ctx, t.hostRef(), @@ -198,14 +203,22 @@ func update[O any]( } // read runs fn on the Stream with no write. -func read[O any](ctx context.Context, t target, view func(*Stream, chasm.Context) (O, error)) (O, error) { +func read[O any]( + ctx context.Context, + t target, + view func(*Stream, chasm.Context) (O, error), +) (O, error) { fn := func(s *Stream, cctx chasm.Context) (O, error) { out, err := view(s, cctx) return out, carry(err) } if !t.nested { - out, err := chasm.ReadComponent(ctx, chasm.NewComponentRef[*Stream](t.key), - func(s *Stream, cctx chasm.Context, _ struct{}) (O, error) { return fn(s, cctx) }, struct{}{}) + out, err := chasm.ReadComponent( + ctx, + chasm.NewComponentRef[*Stream](t.key), + func(s *Stream, cctx chasm.Context, _ struct{}) (O, error) { return fn(s, cctx) }, + struct{}{}, + ) return out, uncarry(err) } out, err := chasm.ReadComponent(ctx, t.hostRef(), @@ -244,9 +257,14 @@ func uncarry(err error) error { func poll(ctx context.Context, t target, done func(*Stream) bool) error { var err error if !t.nested { - _, _, err = chasm.PollComponent(ctx, chasm.NewComponentRef[*Stream](t.key), - func(s *Stream, _ chasm.Context, _ struct{}) (struct{}, bool, error) { return struct{}{}, done(s), nil }, - struct{}{}) + _, _, err = chasm.PollComponent( + ctx, + chasm.NewComponentRef[*Stream](t.key), + func(s *Stream, _ chasm.Context, _ struct{}) (struct{}, bool, error) { + return struct{}{}, done(s), nil + }, + struct{}{}, + ) } else { _, _, err = chasm.PollComponent(ctx, t.hostRef(), func(h Host, cctx chasm.Context, _ struct{}) (struct{}, bool, error) { @@ -266,7 +284,10 @@ type closeChainTaskHandler struct { namespaceRegistry namespace.Registry } -func newCloseChainTaskHandler(config *Config, namespaceRegistry namespace.Registry) *closeChainTaskHandler { +func newCloseChainTaskHandler( + config *Config, + namespaceRegistry namespace.Registry, +) *closeChainTaskHandler { return &closeChainTaskHandler{config: config, namespaceRegistry: namespaceRegistry} } @@ -298,7 +319,11 @@ func (h *closeChainTaskHandler) Execute( } limits := h.config.LimitsFor(ns.Name().String()) first := target{ - key: chasm.ExecutionKey{NamespaceID: ref.NamespaceID, BusinessID: ref.BusinessID, RunID: task.GetFirstRunId()}, + key: chasm.ExecutionKey{ + NamespaceID: ref.NamespaceID, + BusinessID: ref.BusinessID, + RunID: task.GetFirstRunId(), + }, nested: true, host: chasm.WorkflowArchetypeID, } diff --git a/chasm/lib/stream/library.go b/chasm/lib/stream/library.go index c9f79326a72..1c98af93762 100644 --- a/chasm/lib/stream/library.go +++ b/chasm/lib/stream/library.go @@ -36,7 +36,14 @@ func newLibrary( chain *closeChainTaskHandler, readers *readersCheckTaskHandler, ) *library { - return &library{config: config, handler: handler, grpc: grpcHandler, timer: timer, chain: chain, readers: readers} + return &library{ + config: config, + handler: handler, + grpc: grpcHandler, + timer: timer, + chain: chain, + readers: readers, + } } // NewNilLibrary creates a Library with nil handlers, for readers that only decode a Stream. diff --git a/chasm/lib/stream/nexus.go b/chasm/lib/stream/nexus.go index dfe06e2a2f1..3a623f40c94 100644 --- a/chasm/lib/stream/nexus.go +++ b/chasm/lib/stream/nexus.go @@ -22,10 +22,12 @@ const ServiceName = "temporal.api.streamservice.v1.StreamService" func readerOf(options nexus.StartOperationOptions) Reader { reader := Reader{RequestID: options.RequestID, Run: readerRunOf(options.Links)} if options.CallbackURL != "" { - reader.Callback = &commonpb.Callback{Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{ - Url: options.CallbackURL, - Header: options.CallbackHeader, - }}} + reader.Callback = &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{ + Url: options.CallbackURL, + Header: options.CallbackHeader, + }}, + } } return reader } @@ -56,21 +58,34 @@ func nexusError(err error) error { func newNexusService(h *handler) *nexus.Service { service := nexus.NewService(ServiceName) service.MustRegister( - nexus.NewSyncOperation("DescribeStream", - func(ctx context.Context, req *streamservice.DescribeStreamRequest, _ nexus.StartOperationOptions) (*streamservice.DescribeStreamResponse, error) { + nexus.NewSyncOperation( + "DescribeStream", + func( + ctx context.Context, + req *streamservice.DescribeStreamRequest, + _ nexus.StartOperationOptions, + ) (*streamservice.DescribeStreamResponse, error) { resp, err := h.DescribeStream(ctx, req) return resp, nexusError(err) - }), - nexus.NewSyncOperation("CloseStream", - func(ctx context.Context, req *streamservice.CloseStreamRequest, options nexus.StartOperationOptions) (*streamservice.CloseStreamResponse, error) { + }, + ), + nexus.NewSyncOperation( + "CloseStream", + func( + ctx context.Context, + req *streamservice.CloseStreamRequest, + options nexus.StartOperationOptions, + ) (*streamservice.CloseStreamResponse, error) { // A command's start always has the system callback, and a direct call never has // one. A direct call waits for no Workflow Task to commit, so it can't close. if options.CallbackURL == "" { - return nil, nexusError(invalid("A direct System Nexus call can't close a Stream, so use the gRPC CloseStream")) + return nil, nexusError(invalid("A direct System Nexus call can't close a " + + "Stream, so use the gRPC CloseStream")) } resp, err := h.CloseFromRun(ctx, req, readerRunOf(options.Links)) return resp, nexusError(err) - }), + }, + ), &readRecordsOperation{handler: h}, ) return service @@ -80,7 +95,10 @@ func newNexusService(h *handler) *nexus.Service { // async operation that the Stream completes when Records come or the Stream closes, so a // Workflow waits on the server with no timer. type readRecordsOperation struct { - nexus.UnimplementedOperation[*streamservice.ReadRecordsRequest, *streamservice.ReadRecordsResponse] + nexus.UnimplementedOperation[ + *streamservice.ReadRecordsRequest, + *streamservice.ReadRecordsResponse, + ] handler *handler } @@ -105,7 +123,9 @@ func (o *readRecordsOperation) Start( if err != nil { return nil, nexusError(err) } - return &nexus.HandlerStartOperationResultSync[*streamservice.ReadRecordsResponse]{Value: resp}, nil + return &nexus.HandlerStartOperationResultSync[*streamservice.ReadRecordsResponse]{ + Value: resp, + }, nil } resp, err := o.handler.ReadOrPark(ctx, req, readerOf(options)) if err != nil { @@ -126,10 +146,16 @@ func (o *readRecordsOperation) Start( } return &nexus.HandlerStartOperationResultAsync{OperationToken: string(token)}, nil } - return &nexus.HandlerStartOperationResultSync[*streamservice.ReadRecordsResponse]{Value: resp}, nil + return &nexus.HandlerStartOperationResultSync[*streamservice.ReadRecordsResponse]{ + Value: resp, + }, nil } -func (o *readRecordsOperation) Cancel(ctx context.Context, token string, _ nexus.CancelOperationOptions) error { +func (o *readRecordsOperation) Cancel( + ctx context.Context, + token string, + _ nexus.CancelOperationOptions, +) error { var t readToken if err := json.Unmarshal([]byte(token), &t); err != nil { return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid operation token") @@ -206,7 +232,11 @@ func (p processor[R]) ProcessInput( p.setNamespace(req, name) case name: default: - return nil, serviceerror.NewInvalidArgumentf("Namespace in request %q does not match namespace in context %q", req.GetNamespace(), name) + return nil, serviceerror.NewInvalidArgumentf( + "Namespace in request %q does not match namespace in context %q", + req.GetNamespace(), + name, + ) } businessID := ref.GetWorkflowId() if businessID == "" { @@ -225,17 +255,30 @@ func (p processor[R]) ProcessInput( func newNexusServiceProcessor(config *Config) *chasm.NexusServiceProcessor { sp := chasm.NewNexusServiceProcessor(ServiceName) - sp.MustRegisterOperation("ReadRecords", chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.ReadRecordsRequest]{ - config: config, - setNamespace: func(r *streamservice.ReadRecordsRequest, ns string) { r.Namespace = ns }, - })) - sp.MustRegisterOperation("DescribeStream", chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.DescribeStreamRequest]{ - config: config, - setNamespace: func(r *streamservice.DescribeStreamRequest, ns string) { r.Namespace = ns }, - })) - sp.MustRegisterOperation("CloseStream", chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.CloseStreamRequest]{ - config: config, - setNamespace: func(r *streamservice.CloseStreamRequest, ns string) { r.Namespace = ns }, - })) + sp.MustRegisterOperation( + "ReadRecords", + chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.ReadRecordsRequest]{ + config: config, + setNamespace: func(r *streamservice.ReadRecordsRequest, ns string) { r.Namespace = ns }, + }), + ) + sp.MustRegisterOperation( + "DescribeStream", + chasm.NewRegisterableNexusOperationProcessor( + processor[*streamservice.DescribeStreamRequest]{ + config: config, + setNamespace: func(r *streamservice.DescribeStreamRequest, ns string) { + r.Namespace = ns + }, + }, + ), + ) + sp.MustRegisterOperation( + "CloseStream", + chasm.NewRegisterableNexusOperationProcessor(processor[*streamservice.CloseStreamRequest]{ + config: config, + setNamespace: func(r *streamservice.CloseStreamRequest, ns string) { r.Namespace = ns }, + }), + ) return sp } diff --git a/chasm/lib/stream/nexus_test.go b/chasm/lib/stream/nexus_test.go index 701a7fe6559..4309f9cb36c 100644 --- a/chasm/lib/stream/nexus_test.go +++ b/chasm/lib/stream/nexus_test.go @@ -23,7 +23,9 @@ func TestNexusErrorFailsTheOperationWithTheReason(t *testing.T) { require.True(t, ok) require.Equal(t, nexus.OperationStateFailed, opErr.State) - _, ok = nexusError(serviceerror.NewResourceExhausted(enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, "full")).(*nexus.HandlerError) + full := serviceerror.NewResourceExhausted( + enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, "full") + _, ok = nexusError(full).(*nexus.HandlerError) require.True(t, ok) unavailable := serviceerror.NewUnavailable("down") @@ -32,32 +34,72 @@ func TestNexusErrorFailsTheOperationWithTheReason(t *testing.T) { func TestProcessorRoutesByStreamID(t *testing.T) { ns := namespace.NewLocalNamespaceForTest( - &persistencespb.NamespaceInfo{Id: "ns-id", Name: "ns"}, &persistencespb.NamespaceConfig{}, "active") + &persistencespb.NamespaceInfo{ + Id: "ns-id", + Name: "ns", + }, + &persistencespb.NamespaceConfig{}, + "active", + ) collection := dynamicconfig.NewMemoryClient() collection.OverrideSetting(Enabled, true) - sp := newNexusServiceProcessor(ConfigProvider(dynamicconfig.NewCollection(collection, log.NewNoopLogger()))) + sp := newNexusServiceProcessor( + ConfigProvider(dynamicconfig.NewCollection(collection, log.NewNoopLogger())), + ) ctx := chasm.NexusOperationProcessorContext{Namespace: ns, ReserializeInputPayload: true} - input := payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{Stream: &streamservice.StreamRef{Id: "chat-1"}}) + input := payloads.MustEncodeSingle( + &streamservice.ReadRecordsRequest{Stream: &streamservice.StreamRef{Id: "chat-1"}}, + ) result, err := sp.ProcessInput(ctx, "ReadRecords", input) require.NoError(t, err) - require.Equal(t, chasm.NexusOperationRoutingKeyExecution{NamespaceID: "ns-id", BusinessID: "chat-1"}, result.RoutingKey) + require.Equal( + t, + chasm.NexusOperationRoutingKeyExecution{NamespaceID: "ns-id", BusinessID: "chat-1"}, + result.RoutingKey, + ) var routed streamservice.ReadRecordsRequest - require.NoError(t, payloads.Decode(&commonpb.Payloads{Payloads: []*commonpb.Payload{result.ReserializedInputPayload}}, &routed)) + require.NoError( + t, + payloads.Decode( + &commonpb.Payloads{Payloads: []*commonpb.Payload{result.ReserializedInputPayload}}, + &routed, + ), + ) require.Equal(t, "ns", routed.GetNamespace()) // A Workflow Stream is on its Workflow's shard. - result, err = sp.ProcessInput(ctx, "CloseStream", payloads.MustEncodeSingle(&streamservice.CloseStreamRequest{ - Stream: &streamservice.StreamRef{Id: "chat-1", WorkflowId: "wf", FirstExecutionRunId: "first"}, - })) + result, err = sp.ProcessInput( + ctx, + "CloseStream", + payloads.MustEncodeSingle(&streamservice.CloseStreamRequest{ + Stream: &streamservice.StreamRef{ + Id: "chat-1", + WorkflowId: "wf", + FirstExecutionRunId: "first", + }, + }), + ) require.NoError(t, err) - require.Equal(t, chasm.NexusOperationRoutingKeyExecution{NamespaceID: "ns-id", BusinessID: "wf"}, result.RoutingKey) + require.Equal( + t, + chasm.NexusOperationRoutingKeyExecution{NamespaceID: "ns-id", BusinessID: "wf"}, + result.RoutingKey, + ) - _, err = sp.ProcessInput(ctx, "ReadRecords", payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{})) + _, err = sp.ProcessInput( + ctx, + "ReadRecords", + payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{}), + ) require.ErrorContains(t, err, ReasonInvalid) - _, err = sp.ProcessInput(ctx, "ReadRecords", payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{ - Namespace: "other", - Stream: &streamservice.StreamRef{Id: "chat-1"}, - })) + _, err = sp.ProcessInput( + ctx, + "ReadRecords", + payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{ + Namespace: "other", + Stream: &streamservice.StreamRef{Id: "chat-1"}, + }), + ) require.ErrorContains(t, err, "does not match") } diff --git a/chasm/lib/stream/parked_read.go b/chasm/lib/stream/parked_read.go index 97e139dc02f..9a8cda76e23 100644 --- a/chasm/lib/stream/parked_read.go +++ b/chasm/lib/stream/parked_read.go @@ -52,7 +52,10 @@ func (r *ParkedRead) CompletionCallbackMetadata() *callbackspb.CallbackMetadata // GetNexusCompletion gives the page as the result, or the error as the failure of the // operation. -func (r *ParkedRead) GetNexusCompletion(ctx chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { +func (r *ParkedRead) GetNexusCompletion( + ctx chasm.Context, + _ string, +) (nexusrpc.CompleteOperationOptions, error) { opts := nexusrpc.CompleteOperationOptions{StartTime: r.GetStartTime().AsTime()} if r.GetFailure() == nil { result, err := SystemPayload(r.GetResponse()) @@ -70,7 +73,11 @@ func (r *ParkedRead) GetNexusCompletion(ctx chasm.Context, _ string) (nexusrpc.C return opts, nil } -func (r *ParkedRead) resolve(ctx chasm.MutableContext, resp *streamservice.ReadRecordsResponse, err error) { +func (r *ParkedRead) resolve( + ctx chasm.MutableContext, + resp *streamservice.ReadRecordsResponse, + err error, +) { r.Resolved = true r.CreateDeadline = nil if err != nil { @@ -88,19 +95,25 @@ func (r *ParkedRead) resolve(ctx chasm.MutableContext, resp *streamservice.ReadR func FailureOf(err error) *failurepb.Failure { if errors.Is(err, errCanceled) { return &failurepb.Failure{ - Message: "The read was canceled", - FailureInfo: &failurepb.Failure_CanceledFailureInfo{CanceledFailureInfo: &failurepb.CanceledFailureInfo{}}, + Message: "The read was canceled", + FailureInfo: &failurepb.Failure_CanceledFailureInfo{ + CanceledFailureInfo: &failurepb.CanceledFailureInfo{}, + }, } } reason := ReasonOf(err) - details, _ := sdkconverter.PreferProtoDataConverter.ToPayloads(&streamservice.StreamErrorDetails{Reason: reason}) + details, _ := sdkconverter.PreferProtoDataConverter.ToPayloads( + &streamservice.StreamErrorDetails{Reason: reason}, + ) return &failurepb.Failure{ Message: err.Error(), - FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ - Type: reason, - NonRetryable: true, - Details: details, - }}, + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: reason, + NonRetryable: true, + Details: details, + }, + }, } } diff --git a/chasm/lib/stream/producer.go b/chasm/lib/stream/producer.go index c745d215bfb..7706dec9d47 100644 --- a/chasm/lib/stream/producer.go +++ b/chasm/lib/stream/producer.go @@ -34,22 +34,33 @@ func (s *Stream) admit( } known := s.producer(ctx, producerKey(producer)) if known != nil && CompareEpochs(producer.GetEpoch(), known.GetEpoch()) < 0 { - return false, NewError(ReasonFenced, "The epoch is lower than the newest one of the Producer") + return false, NewError( + ReasonFenced, + "The epoch is lower than the newest one of the Producer", + ) } run := s.run(ctx, producer.GetRunId()) task := taskOf(producer.GetEpoch()) switch producer.GetKind() { case streamservice.PRODUCER_KIND_LOCAL_ACTIVITY: - if run != nil && (slices.ContainsFunc(run.GetWithdrawn(), func(t *streamservice.WorkflowTaskEpoch) bool { - return compareTasks(t, task) == 0 - }) || (run.GetLastTask() != nil && len(withdrawTasks( - []*streamservice.WorkflowTaskEpoch{task}, run.GetLastTask(), run.GetLastCompleted(), run.GetLastFailed(), - ).withdrawn) > 0)) { - return false, NewError(ReasonFenced, "A Superseded Record withdrew the Workflow Task of the call") + withdrawn := slices.ContainsFunc(run.GetWithdrawn(), + func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }) + if run.GetLastTask() != nil { + index := []*streamservice.WorkflowTaskEpoch{task} + split := withdrawTasks(index, run.GetLastTask(), run.GetLastCompleted(), + run.GetLastFailed()) + withdrawn = withdrawn || len(split.withdrawn) > 0 + } + if withdrawn { + return false, NewError( + ReasonFenced, + "A Superseded Record withdrew the Workflow Task of the call", + ) } return newEpoch(producer, known) || s.withdrawsRun(ctx, w.WithdrawRun), nil case streamservice.PRODUCER_KIND_WORKFLOW: - flagged := !w.Defer && len(withdrawTasks(run.GetIndex(), task, w.PreviousCompleted, w.FailedTasks).withdrawn) > 0 + split := withdrawTasks(run.GetIndex(), task, w.PreviousCompleted, w.FailedTasks) + flagged := !w.Defer && len(split.withdrawn) > 0 return flagged || s.withdrawsRun(ctx, w.WithdrawRun), nil default: return newEpoch(producer, known) || s.withdrawsRun(ctx, w.WithdrawRun), nil @@ -88,7 +99,10 @@ func (s *Stream) settle( } if wroteRecords { run.Wrote = true - if !slices.ContainsFunc(run.Index, func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }) { + if !slices.ContainsFunc( + run.Index, + func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }, + ) { run.Index = append(run.Index, task) } } @@ -180,7 +194,11 @@ func (s *Stream) settleOrphans( run := s.runForWrite(ctx, producer.GetRunId()) run.OpenLocal = split.kept task := taskOf(producer.GetEpoch()) - if wroteRecords && !slices.ContainsFunc(run.OpenLocal, func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }) { + if wroteRecords && + !slices.ContainsFunc( + run.OpenLocal, + func(t *streamservice.WorkflowTaskEpoch) bool { return compareTasks(t, task) == 0 }, + ) { run.OpenLocal = append(run.OpenLocal, task) } run.Withdrawn = append(run.Withdrawn, split.withdrawn...) diff --git a/chasm/lib/stream/read.go b/chasm/lib/stream/read.go index 9c35f841e4a..008170561fe 100644 --- a/chasm/lib/stream/read.go +++ b/chasm/lib/stream/read.go @@ -36,7 +36,12 @@ func (s *Stream) StartOffset(req *streamservice.ReadRecordsRequest) (int64, erro from = s.NewestOffset + 1 } if from > s.NewestOffset+1 { - return 0, NewError(ReasonOffset, "The offset %d is past the newest offset + 1, %d", from, s.NewestOffset+1) + return 0, NewError( + ReasonOffset, + "The offset %d is past the newest offset + 1, %d", + from, + s.NewestOffset+1, + ) } return from, nil } @@ -265,13 +270,19 @@ func passes(req *streamservice.ReadRecordsRequest, topic string) bool { // wakeAnswer is the answer of a wake_only read: no Records, the read's start as its next // offset, and whether the Stream is closed. func (s *Stream) wakeAnswer(start int64) *streamservice.ReadRecordsResponse { - return &streamservice.ReadRecordsResponse{NextOffset: start, Done: s.Phase == streampb.STREAM_PHASE_CLOSED} + return &streamservice.ReadRecordsResponse{ + NextOffset: start, + Done: s.Phase == streampb.STREAM_PHASE_CLOSED, + } } // afterClose gives the Superseded items written after the close from the offset from, and the // offset after the last entry, zero if none. A Workflow Task that completes after the close // still withdraws, and a reader stops at the close Record, so a page puts them before it. -func (s *Stream) afterClose(ctx chasm.Context, from, end int64) ([]*streamservice.StreamRecord, int64) { +func (s *Stream) afterClose( + ctx chasm.Context, + from, end int64, +) ([]*streamservice.StreamRecord, int64) { var records []*streamservice.StreamRecord var after int64 for _, info := range s.Index[s.firstBatchAt(from):] { @@ -332,7 +343,10 @@ func StartForRead(ctx chasm.MutableContext, in ReadInput) (*Stream, error) { // ReadOrPark answers a read at once, but parks a wake_only read that finds no Record until the // Stream can answer it. It answers nil for a parked read. A wake's answer has no Records, so a // parked read keeps no page, though one append wakes every parked read in one transaction. -func (s *Stream) ReadOrPark(ctx chasm.MutableContext, in ReadInput) (*streamservice.ReadRecordsResponse, error) { +func (s *Stream) ReadOrPark( + ctx chasm.MutableContext, + in ReadInput, +) (*streamservice.ReadRecordsResponse, error) { s.applyDue(ctx, in.Limits) if field, ok := s.Reads[in.RequestID]; ok { read := field.Get(ctx) @@ -399,7 +413,9 @@ func (s *Stream) ReadOrPark(ctx chasm.MutableContext, in ReadInput) (*streamserv s.Reads = make(chasm.Map[string, *ParkedRead]) } s.Reads[in.RequestID] = chasm.NewComponentField(ctx, read) - if err := callback.Attach(ctx, read, in.RequestID, timestamppb.New(now), []*commonpb.Callback{in.Callback}); err != nil { + callbacks := []*commonpb.Callback{in.Callback} + err := callback.Attach(ctx, read, in.RequestID, timestamppb.New(now), callbacks) + if err != nil { return nil, err } s.finish(ctx, in.Limits) @@ -459,7 +475,10 @@ func (s *Stream) wake( } // resolveReads completes each waiting read for which decide is done. -func (s *Stream) resolveReads(ctx chasm.MutableContext, decide func(*ParkedRead) (readOutcome, bool)) { +func (s *Stream) resolveReads( + ctx chasm.MutableContext, + decide func(*ParkedRead) (readOutcome, bool), +) { for _, field := range s.Reads { read := field.Get(ctx) if read.GetResolved() { @@ -491,7 +510,8 @@ func (s *Stream) pruneReads(ctx chasm.MutableContext) { delivered := true for _, cb := range read.Callbacks { status := cb.Get(ctx).GetStatus() - if status != callbackspb.CALLBACK_STATUS_SUCCEEDED && status != callbackspb.CALLBACK_STATUS_FAILED { + if status != callbackspb.CALLBACK_STATUS_SUCCEEDED && + status != callbackspb.CALLBACK_STATUS_FAILED { delivered = false break } @@ -516,8 +536,12 @@ func parkWaits( } // ReadTimeout is how long a frontend read waits, bounded by the crate's limit. -func ReadTimeout(req *streamservice.ReadRecordsRequest) (wait time.Duration, createWait time.Duration) { - return min(max(req.GetWait().AsDuration(), 0), MaxWait), min(max(req.GetCreateWait().AsDuration(), 0), MaxWait) +func ReadTimeout( + req *streamservice.ReadRecordsRequest, +) (wait time.Duration, createWait time.Duration) { + wait = min(max(req.GetWait().AsDuration(), 0), MaxWait) + createWait = min(max(req.GetCreateWait().AsDuration(), 0), MaxWait) + return wait, createWait } // keepWakeFloor keeps the floor of a wake's Workflow in the transaction that takes the wake, diff --git a/chasm/lib/stream/readers.go b/chasm/lib/stream/readers.go index 71df5924bdb..f1645fed287 100644 --- a/chasm/lib/stream/readers.go +++ b/chasm/lib/stream/readers.go @@ -82,7 +82,12 @@ func (s *Stream) suggestsContinueAsNew(ctx chasm.Context, runID string, limits L // AddReader keeps the floor of a Workflow run that reads from from, in the transaction that // answers the read. No trim passes the floor while the run is open. -func (s *Stream) AddReader(ctx chasm.MutableContext, run ReaderRun, from int64, limits Limits) error { +func (s *Stream) AddReader( + ctx chasm.MutableContext, + run ReaderRun, + from int64, + limits Limits, +) error { floor, ok := s.readerFloor(ctx, run.RunID, from) if !ok { return nil @@ -126,7 +131,11 @@ func newReadersCheckTaskHandler( namespaceRegistry namespace.Registry, historyClient resource.HistoryClient, ) *readersCheckTaskHandler { - return &readersCheckTaskHandler{config: config, namespaceRegistry: namespaceRegistry, historyClient: historyClient} + return &readersCheckTaskHandler{ + config: config, + namespaceRegistry: namespaceRegistry, + historyClient: historyClient, + } } func (h *readersCheckTaskHandler) Validate( @@ -135,7 +144,8 @@ func (h *readersCheckTaskHandler) Validate( _ chasm.TaskInvocation, task *streampb.StreamReadersCheckTask, ) (bool, error) { - return s.GetReadersCheckTime() != nil && s.GetReadersCheckTime().AsTime().Equal(task.GetDeadline().AsTime()), nil + return s.GetReadersCheckTime() != nil && + s.GetReadersCheckTime().AsTime().Equal(task.GetDeadline().AsTime()), nil } func (h *readersCheckTaskHandler) Execute( @@ -157,7 +167,12 @@ func (h *readersCheckTaskHandler) Execute( } var closed []string for runID, workflowID := range readers { - open, err := runOpen(ctx, h.historyClient, ref.NamespaceID, ReaderRun{WorkflowID: workflowID, RunID: runID}) + open, err := runOpen( + ctx, + h.historyClient, + ref.NamespaceID, + ReaderRun{WorkflowID: workflowID, RunID: runID}, + ) if err != nil { return err } @@ -181,9 +196,14 @@ func (h *readersCheckTaskHandler) Execute( return err } -// runOpen says whether a Workflow run is still open. A deleted or unknown run is closed. The run may be on -// another shard, so it goes through the history client. -func runOpen(ctx context.Context, historyClient resource.HistoryClient, namespaceID string, run ReaderRun) (bool, error) { +// runOpen says whether a Workflow run is still open. A deleted or unknown run is closed. The run +// may be on another shard, so it goes through the history client. +func runOpen( + ctx context.Context, + historyClient resource.HistoryClient, + namespaceID string, + run ReaderRun, +) (bool, error) { resp, err := historyClient.GetMutableState(ctx, &historyservice.GetMutableStateRequest{ NamespaceId: namespaceID, Execution: &commonpb.WorkflowExecution{WorkflowId: run.WorkflowID, RunId: run.RunID}, diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index 387de5f068b..83294e5822c 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -213,7 +213,10 @@ type AppendInput struct { // Append writes one batch of one Producer, with the checks and the withdrawal of the crate's // append script in one transition. The same epoch and sequence answers the first offsets // again, so a retried batch goes in once. -func (s *Stream) Append(ctx chasm.MutableContext, in AppendInput) (*streamservice.AppendRecordsResponse, error) { +func (s *Stream) Append( + ctx chasm.MutableContext, + in AppendInput, +) (*streamservice.AppendRecordsResponse, error) { req := in.Request producer := req.GetProducer() if req.GetSequence() < 1 { @@ -235,7 +238,10 @@ func (s *Stream) Append(ctx chasm.MutableContext, in AppendInput) (*streamservic return nil, NewError(ReasonSeqDivergent, "The sequence %d has other content than its batch", req.GetSequence()) } - return &streamservice.AppendRecordsResponse{FirstOffset: known.GetFirstOffset(), LastOffset: known.GetLastOffset()}, nil + return &streamservice.AppendRecordsResponse{ + FirstOffset: known.GetFirstOffset(), + LastOffset: known.GetLastOffset(), + }, nil } if req.GetSequence() < known.GetSequence() { return nil, NewError(ReasonSequence, @@ -244,8 +250,11 @@ func (s *Stream) Append(ctx chasm.MutableContext, in AppendInput) (*streamservic } // A Workflow batch with no Records only settles its task facts, which a closed Stream // still takes, as the Superseded Records after the close. - deferred := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && req.GetDeferWithdrawal() - settles := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && len(req.GetRecords()) == 0 && !deferred + deferred := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && + req.GetDeferWithdrawal() + settles := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && + len(req.GetRecords()) == 0 && + !deferred if s.Phase == streampb.STREAM_PHASE_CLOSED && !settles { return nil, NewError(ReasonClosed, "The Stream is closed") } @@ -332,7 +341,11 @@ func BatchDigest(records []*streamservice.AppendRecord) []byte { // batchEntries are the entries of one write: the orphans' Superseded Record, the batch's own // Superseded Record, then the Records. -func batchEntries(req *streamservice.AppendRecordsRequest, w Withdrawal, flagged, orphans bool) []*streamservice.StreamRecord { +func batchEntries( + req *streamservice.AppendRecordsRequest, + w Withdrawal, + flagged, orphans bool, +) []*streamservice.StreamRecord { producer := req.GetProducer() var entries []*streamservice.StreamRecord // The orphans go in an entry of their own, since the batch's first entry may carry its own @@ -358,16 +371,30 @@ func batchEntries(req *streamservice.AppendRecordsRequest, w Withdrawal, flagged // readers keep their Records, or one that its Workflow can't take. func (s *Stream) checkSize(ctx chasm.MutableContext, size int64, limits Limits) error { if size > limits.MaxBatchBytes { - return NewError(ReasonTooLarge, "The batch has %d bytes, more than %d", size, limits.MaxBatchBytes) + return NewError( + ReasonTooLarge, + "The batch has %d bytes, more than %d", + size, + limits.MaxBatchBytes, + ) } if kept := s.keptBytes(); kept+size > limits.MaxStoredBytes { - return NewError(ReasonTooLarge, "The Stream keeps %d bytes for its readers, and the cap is %d", kept, limits.MaxStoredBytes) + return NewError( + ReasonTooLarge, + "The Stream keeps %d bytes for its readers, and the cap is %d", + kept, + limits.MaxStoredBytes, + ) } // The Records of a Workflow Stream are in the Workflow's mutable state, which terminates the // run past its own limit, so the write fails first. if limits.MaxWorkflowStateBytes > 0 && int64(ctx.ExecutionInfo().ApproximateStateSize)+size > limits.MaxWorkflowStateBytes { - return NewError(ReasonTooLarge, "The Workflow state would pass %d bytes", limits.MaxWorkflowStateBytes) + return NewError( + ReasonTooLarge, + "The Workflow state would pass %d bytes", + limits.MaxWorkflowStateBytes, + ) } return nil } @@ -384,7 +411,10 @@ type CloseInput struct { // Close writes the close Record, with the Producer checks and the withdrawal of a batch. A // second close does nothing. -func (s *Stream) Close(ctx chasm.MutableContext, in CloseInput) (*streamservice.CloseStreamResponse, error) { +func (s *Stream) Close( + ctx chasm.MutableContext, + in CloseInput, +) (*streamservice.CloseStreamResponse, error) { req := in.Request producer := req.GetProducer() status, err := callerStatus(req) @@ -410,7 +440,11 @@ func (s *Stream) Close(ctx chasm.MutableContext, in CloseInput) (*streamservice. return nil, NewError(ReasonFenced, "A Workflow retry withdrew the run of the Producer") } } else { - w := Withdrawal{PreviousCompleted: req.GetPreviousCompletedTask(), FailedTasks: req.GetFailedTasks(), WithdrawRun: req.GetWithdrawRun()} + w := Withdrawal{ + PreviousCompleted: req.GetPreviousCompletedTask(), + FailedTasks: req.GetFailedTasks(), + WithdrawRun: req.GetWithdrawRun(), + } flagged, err := s.admit(ctx, producer, w) if err != nil { return nil, err @@ -481,7 +515,10 @@ func (s *Stream) writable() error { // withdrawOnCreate applies a retry run's withdraw_run at its first create. A create has no task // facts, so it withdraws no task, and its Superseded Record has no epoch, as the crate's. -func (s *Stream) withdrawOnCreate(ctx chasm.MutableContext, req *streamservice.CreateStreamRequest) { +func (s *Stream) withdrawOnCreate( + ctx chasm.MutableContext, + req *streamservice.CreateStreamRequest, +) { run := req.GetWithdrawRun() if run == "" || s.runFenced(run) { return @@ -494,7 +531,10 @@ func (s *Stream) withdrawOnCreate(ctx chasm.MutableContext, req *streamservice.C } // write gives the entries their offsets and stores them as one batch. -func (s *Stream) write(ctx chasm.MutableContext, entries []*streamservice.StreamRecord) (first int64, last int64) { +func (s *Stream) write( + ctx chasm.MutableContext, + entries []*streamservice.StreamRecord, +) (first int64, last int64) { if len(entries) == 0 { return 0, 0 } @@ -510,7 +550,10 @@ func (s *Stream) write(ctx chasm.MutableContext, entries []*streamservice.Stream } s.Batches[first] = chasm.NewDataField(ctx, batch) size := int64(proto.Size(batch)) - s.Index = append(s.Index, &streampb.BatchInfo{FirstOffset: first, LastOffset: last, Bytes: size}) + s.Index = append( + s.Index, + &streampb.BatchInfo{FirstOffset: first, LastOffset: last, Bytes: size}, + ) s.StoredBytes += size s.NewestOffset = last return first, last @@ -630,7 +673,8 @@ func (s *Stream) applyDue(ctx chasm.MutableContext, limits Limits) { switch s.Phase { case streampb.STREAM_PHASE_PENDING: s.resolveReads(ctx, func(read *ParkedRead) (readOutcome, bool) { - if deadline := read.GetCreateDeadline(); deadline != nil && !deadline.AsTime().After(now) { + if deadline := read.GetCreateDeadline(); deadline != nil && + !deadline.AsTime().After(now) { return readOutcome{err: notFound()}, true } return readOutcome{}, false @@ -644,7 +688,8 @@ func (s *Stream) applyDue(ctx chasm.MutableContext, limits Limits) { default: } // A reader's replay needs its Records, so the Stream expires only when it has no reader. - if s.Phase == streampb.STREAM_PHASE_CLOSED && !s.closedExpiryAt().After(now) && len(s.Readers) == 0 { + if s.Phase == streampb.STREAM_PHASE_CLOSED && !s.closedExpiryAt().After(now) && + len(s.Readers) == 0 { s.expire(ctx) return } @@ -713,12 +758,16 @@ func (s *Stream) nextDeadline(ctx chasm.Context, limits Limits) (time.Time, bool return deadline, false } for _, field := range s.Reads { - if read, ok := field.TryGet(ctx); ok && !read.GetResolved() && read.GetWaitDeadline() != nil { + if read, ok := field.TryGet(ctx); ok && !read.GetResolved() && + read.GetWaitDeadline() != nil { consider(read.GetWaitDeadline().AsTime()) } } - if retention := s.GetRetention().AsDuration(); retention > 0 && len(s.Index) > 0 && !s.kept(s.Index[0]) { - consider(time.UnixMilli(s.Index[0].GetLastOffset() >> seqBits).Add(retention + time.Millisecond)) + if retention := s.GetRetention().AsDuration(); retention > 0 && len(s.Index) > 0 && + !s.kept(s.Index[0]) { + consider( + time.UnixMilli(s.Index[0].GetLastOffset() >> seqBits).Add(retention + time.Millisecond), + ) } return deadline, found } @@ -753,7 +802,10 @@ func (s *Stream) finish(ctx chasm.MutableContext, limits Limits) { // producerKey keys the Producer table by kind, id and run. func producerKey(producer *streamservice.Producer) string { - return strings.Join([]string{producer.GetKind().String(), producer.GetId(), producer.GetRunId()}, "|") + return strings.Join( + []string{producer.GetKind().String(), producer.GetId(), producer.GetRunId()}, + "|", + ) } // CompareEpochs compares two epochs field by field, in the order of the crate. The order is the diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index 48ae89d6ed9..7d6421518b4 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -42,7 +42,10 @@ func newHarness(t *testing.T) *harness { require.NoError(t, registry.Register(&chasm.CoreLibrary{})) require.NoError(t, registry.Register(callback.NewNilLibrary())) config := ConfigProvider(dynamicconfig.NewNoopCollection()) - require.NoError(t, registry.Register(newLibrary(config, nil, nil, newTimerTaskHandler(config), nil, nil))) + require.NoError( + t, + registry.Register(newLibrary(config, nil, nil, newTimerTaskHandler(config), nil, nil)), + ) ts := clock.NewEventTimeSource() ts.Update(time.Date(2026, 10, 22, 0, 0, 0, 0, time.UTC)) engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(ts)) @@ -61,16 +64,27 @@ func (h *harness) ref() chasm.ComponentRef { } func (h *harness) create(req *streamservice.CreateStreamRequest) (CreateOutput, error) { - result, err := chasm.UpdateWithStartExecution(h.ctx, h.key, StartStream, (*Stream).Create, + result, err := chasm.UpdateWithStartExecution( + h.ctx, + h.key, + StartStream, + (*Stream).Create, CreateInput{Request: req, Limits: h.limits}, - chasm.WithBusinessIDPolicy(chasm.BusinessIDReusePolicyRejectDuplicate, chasm.BusinessIDConflictPolicyUseExisting), + chasm.WithBusinessIDPolicy( + chasm.BusinessIDReusePolicyRejectDuplicate, + chasm.BusinessIDConflictPolicyUseExisting, + ), ) out := result.UpdateOutput out.Created = out.Created || result.Created return out, err } -func (h *harness) append(producer *streamservice.Producer, sequence int64, topics ...string) (*streamservice.AppendRecordsResponse, error) { +func (h *harness) append( + producer *streamservice.Producer, + sequence int64, + topics ...string, +) (*streamservice.AppendRecordsResponse, error) { req := &streamservice.AppendRecordsRequest{Producer: producer, Sequence: sequence} for _, topic := range topics { req.Records = append(req.Records, &streamservice.AppendRecord{ @@ -81,8 +95,15 @@ func (h *harness) append(producer *streamservice.Producer, sequence int64, topic return h.appendRequest(req) } -func (h *harness) appendRequest(req *streamservice.AppendRecordsRequest) (*streamservice.AppendRecordsResponse, error) { - resp, _, err := chasm.UpdateComponent(h.ctx, h.ref(), (*Stream).Append, AppendInput{Request: req, Limits: h.limits}) +func (h *harness) appendRequest( + req *streamservice.AppendRecordsRequest, +) (*streamservice.AppendRecordsResponse, error) { + resp, _, err := chasm.UpdateComponent( + h.ctx, + h.ref(), + (*Stream).Append, + AppendInput{Request: req, Limits: h.limits}, + ) return resp, err } @@ -101,8 +122,12 @@ func (h *harness) close(result string) error { func (h *harness) commandClose(run, result string) error { _, _, err := chasm.UpdateComponent(h.ctx, h.ref(), (*Stream).Close, CloseInput{ Request: &streamservice.CloseStreamRequest{ - Producer: &streamservice.Producer{Kind: streamservice.PRODUCER_KIND_WORKFLOW, Id: "wf:" + run, RunId: run}, - Result: &commonpb.Payload{Data: []byte(result)}, + Producer: &streamservice.Producer{ + Kind: streamservice.PRODUCER_KIND_WORKFLOW, + Id: "wf:" + run, + RunId: run, + }, + Result: &commonpb.Payload{Data: []byte(result)}, }, Limits: h.limits, Command: true, @@ -110,31 +135,51 @@ func (h *harness) commandClose(run, result string) error { return err } -func (h *harness) read(req *streamservice.ReadRecordsRequest) (*streamservice.ReadRecordsResponse, error) { - return chasm.ReadComponent(h.ctx, h.ref(), - func(s *Stream, ctx chasm.Context, req *streamservice.ReadRecordsRequest) (*streamservice.ReadRecordsResponse, error) { +func (h *harness) read( + req *streamservice.ReadRecordsRequest, +) (*streamservice.ReadRecordsResponse, error) { + return chasm.ReadComponent( + h.ctx, + h.ref(), + func( + s *Stream, + ctx chasm.Context, + req *streamservice.ReadRecordsRequest, + ) (*streamservice.ReadRecordsResponse, error) { from, err := s.StartOffset(req) if err != nil { return nil, err } resp, _, err := s.Page(ctx, req, from) return resp, err - }, req) + }, + req, + ) } -func (h *harness) readOrPark(requestID string, req *streamservice.ReadRecordsRequest) (*streamservice.ReadRecordsResponse, error) { - result, err := chasm.UpdateWithStartExecution(h.ctx, h.key, +func (h *harness) readOrPark( + requestID string, + req *streamservice.ReadRecordsRequest, +) (*streamservice.ReadRecordsResponse, error) { + result, err := chasm.UpdateWithStartExecution( + h.ctx, + h.key, StartForRead, (*Stream).ReadOrPark, ReadInput{ Request: req, RequestID: requestID, - Callback: &commonpb.Callback{Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{ - Url: chasm.NexusCompletionHandlerURL, - }}}, + Callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{ + Url: chasm.NexusCompletionHandlerURL, + }}, + }, Limits: h.limits, }, - chasm.WithBusinessIDPolicy(chasm.BusinessIDReusePolicyRejectDuplicate, chasm.BusinessIDConflictPolicyUseExisting), + chasm.WithBusinessIDPolicy( + chasm.BusinessIDReusePolicyRejectDuplicate, + chasm.BusinessIDConflictPolicyUseExisting, + ), ) return result.UpdateOutput, err } @@ -210,7 +255,12 @@ func TestCreateIsIdempotentForOneRetention(t *testing.T) { require.NoError(t, err) require.Equal(t, CreateOutput{Status: streamservice.STREAM_STATUS_OPEN}, out) - _, err = h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Retention: durationpb.New(time.Hour)}) + _, err = h.create( + &streamservice.CreateStreamRequest{ + StreamId: "chat-1", + Retention: durationpb.New(time.Hour), + }, + ) require.ErrorContains(t, err, ReasonAlreadyExists) } @@ -229,11 +279,18 @@ func TestAppendAndRead(t *testing.T) { require.Equal(t, []string{"turns", "tool/search", "other"}, topics(all)) require.Equal(t, first.GetLastOffset()+1, all.GetNextOffset()) - filtered, err := h.read(&streamservice.ReadRecordsRequest{Topics: []string{"turns"}, TopicPrefixes: []string{"tool/"}}) + filtered, err := h.read( + &streamservice.ReadRecordsRequest{ + Topics: []string{"turns"}, + TopicPrefixes: []string{"tool/"}, + }, + ) require.NoError(t, err) require.Equal(t, []string{"turns", "tool/search"}, topics(filtered)) - page, err := h.read(&streamservice.ReadRecordsRequest{FromOffset: first.GetFirstOffset() + 1, MaxRecords: 1}) + page, err := h.read( + &streamservice.ReadRecordsRequest{FromOffset: first.GetFirstOffset() + 1, MaxRecords: 1}, + ) require.NoError(t, err) require.Equal(t, []string{"tool/search"}, topics(page)) require.Equal(t, first.GetFirstOffset()+2, page.GetNextOffset()) @@ -395,7 +452,9 @@ func TestSecondCloseChecksTheOwner(t *testing.T) { // A command close has no epoch, so the epoch of the run's earlier writes doesn't fence it. func TestCommandCloseOfTheOwnerRun(t *testing.T) { h := newHarness(t) - _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Owner: workflowProducer(1)}) + _, err := h.create( + &streamservice.CreateStreamRequest{StreamId: "chat-1", Owner: workflowProducer(1)}, + ) require.NoError(t, err) _, err = h.append(workflowProducer(3), 1, "a") require.NoError(t, err) @@ -433,11 +492,20 @@ func TestLocalActivityCantCloseAWorkflowStream(t *testing.T) { func TestCommandCloseOfAWithdrawnRunIsFenced(t *testing.T) { h := newHarness(t) - _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Owner: workflowProducer(1)}) + _, err := h.create( + &streamservice.CreateStreamRequest{StreamId: "chat-1", Owner: workflowProducer(1)}, + ) require.NoError(t, err) retry := workflowProducer(3) retry.Id, retry.RunId = "wf:retry", "retry" - _, err = h.appendRequest(&streamservice.AppendRecordsRequest{Producer: retry, Sequence: 1, Records: records("a"), WithdrawRun: "run"}) + _, err = h.appendRequest( + &streamservice.AppendRecordsRequest{ + Producer: retry, + Sequence: 1, + Records: records("a"), + WithdrawRun: "run", + }, + ) require.NoError(t, err) require.ErrorContains(t, h.commandClose("run", "done"), ReasonFenced) @@ -445,7 +513,12 @@ func TestCommandCloseOfAWithdrawnRunIsFenced(t *testing.T) { func TestRetentionDropsOldRecordsAndExpiresTheStream(t *testing.T) { h := newHarness(t) - _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Retention: durationpb.New(time.Hour)}) + _, err := h.create( + &streamservice.CreateStreamRequest{ + StreamId: "chat-1", + Retention: durationpb.New(time.Hour), + }, + ) require.NoError(t, err) old, err := h.append(activityProducer(1), 1, "old") require.NoError(t, err) @@ -463,7 +536,12 @@ func TestRetentionDropsOldRecordsAndExpiresTheStream(t *testing.T) { h.advance(time.Hour) _, err = h.read(&streamservice.ReadRecordsRequest{}) require.ErrorContains(t, err, ReasonExpired) - out, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1", Retention: durationpb.New(time.Hour)}) + out, err := h.create( + &streamservice.CreateStreamRequest{ + StreamId: "chat-1", + Retention: durationpb.New(time.Hour), + }, + ) require.Error(t, err) require.Equal(t, CreateOutput{}, out) } @@ -489,7 +567,11 @@ func TestParkedReadWakesOnAppend(t *testing.T) { read := h.parked("read-1") require.True(t, read.GetResolved()) require.Empty(t, read.GetResponse().GetRecords()) - require.Equal(t, []callbackspb.CallbackStatus{callbackspb.CALLBACK_STATUS_SCHEDULED}, read.callbacks) + require.Equal( + t, + []callbackspb.CallbackStatus{callbackspb.CALLBACK_STATUS_SCHEDULED}, + read.callbacks, + ) again, err := h.readOrPark("read-1", wake) require.NoError(t, err) @@ -573,9 +655,13 @@ func workflowProducer(started int64) *streamservice.Producer { func localActivityProducer(started int64) *streamservice.Producer { return &streamservice.Producer{ - Kind: streamservice.PRODUCER_KIND_LOCAL_ACTIVITY, - Id: "la:run:1", - Epoch: &streamservice.ProducerEpoch{TaskStartedEventId: started, TaskAttempt: 1, Attempt: 1}, + Kind: streamservice.PRODUCER_KIND_LOCAL_ACTIVITY, + Id: "la:run:1", + Epoch: &streamservice.ProducerEpoch{ + TaskStartedEventId: started, + TaskAttempt: 1, + Attempt: 1, + }, RunId: "run", } } @@ -583,7 +669,10 @@ func localActivityProducer(started int64) *streamservice.Producer { func records(topics ...string) []*streamservice.AppendRecord { var out []*streamservice.AppendRecord for _, topic := range topics { - out = append(out, &streamservice.AppendRecord{Topic: topic, Body: &commonpb.Payload{Data: []byte(topic)}}) + out = append( + out, + &streamservice.AppendRecord{Topic: topic, Body: &commonpb.Payload{Data: []byte(topic)}}, + ) } return out } @@ -626,7 +715,14 @@ func TestWithdrawRunFencesThePreviousRun(t *testing.T) { retry := workflowProducer(3) retry.Id, retry.RunId = "wf:retry", "retry" - _, err = h.appendRequest(&streamservice.AppendRecordsRequest{Producer: retry, Sequence: 1, Records: records("b"), WithdrawRun: "run"}) + _, err = h.appendRequest( + &streamservice.AppendRecordsRequest{ + Producer: retry, + Sequence: 1, + Records: records("b"), + WithdrawRun: "run", + }, + ) require.NoError(t, err) _, err = h.append(workflowProducer(9), 1, "c") require.ErrorContains(t, err, ReasonFenced) @@ -686,9 +782,14 @@ func TestReaderFloorKeepsItsRecords(t *testing.T) { b, err := h.append(activityProducer(1), 2, "b") require.NoError(t, err) setReaders := func(fn func(*Stream, chasm.MutableContext) error) { - _, _, err := chasm.UpdateComponent(h.ctx, h.ref(), func(s *Stream, mctx chasm.MutableContext, _ struct{}) (struct{}, error) { - return struct{}{}, fn(s, mctx) - }, struct{}{}) + _, _, err := chasm.UpdateComponent( + h.ctx, + h.ref(), + func(s *Stream, mctx chasm.MutableContext, _ struct{}) (struct{}, error) { + return struct{}{}, fn(s, mctx) + }, + struct{}{}, + ) require.NoError(t, err) } setReaders(func(s *Stream, mctx chasm.MutableContext) error { @@ -697,9 +798,13 @@ func TestReaderFloorKeepsItsRecords(t *testing.T) { b.GetFirstOffset(), h.limits) }) - big := &streamservice.AppendRecordsRequest{Producer: activityProducer(1), Sequence: 3, Records: []*streamservice.AppendRecord{ - {Topic: "c", Body: &commonpb.Payload{Data: make([]byte, 300)}}, - }} + big := &streamservice.AppendRecordsRequest{ + Producer: activityProducer(1), + Sequence: 3, + Records: []*streamservice.AppendRecord{ + {Topic: "c", Body: &commonpb.Payload{Data: make([]byte, 300)}}, + }, + } _, err = h.appendRequest(big) require.ErrorContains(t, err, ReasonTooLarge) @@ -781,7 +886,11 @@ func TestReaderFloorKeepsTheRecordsOfAClientStream(t *testing.T) { // localActivityWrite is a write of the Local Activity call of the run, dispatched by task // started, with the run's last completed task (0 before the first), as the crate's S110 test has. -func localActivityWrite(call string, started, completed int64, topic string) *streamservice.AppendRecordsRequest { +func localActivityWrite( + call string, + started, completed int64, + topic string, +) *streamservice.AppendRecordsRequest { return &streamservice.AppendRecordsRequest{ Producer: &streamservice.Producer{ Kind: streamservice.PRODUCER_KIND_LOCAL_ACTIVITY, @@ -816,9 +925,13 @@ func TestLocalActivityOfATaskThatNeverCompletedIsWithdrawnByTheNextWrite(t *test require.Equal(t, []string{"done", "orphan", "superseded", "other"}, topics(read)) require.Equal(t, read.GetRecords()[3].GetOffset(), other.GetFirstOffset()) require.Equal(t, &streamservice.Supersession{ - ProducerKind: streamservice.PRODUCER_KIND_WORKFLOW, - ProducerId: "wf:run", - Epoch: &streamservice.ProducerEpoch{TaskStartedEventId: 4, TaskAttempt: 1, TaskStartedTime: timestamppb.New(time.Unix(104, 0))}, + ProducerKind: streamservice.PRODUCER_KIND_WORKFLOW, + ProducerId: "wf:run", + Epoch: &streamservice.ProducerEpoch{ + TaskStartedEventId: 4, + TaskAttempt: 1, + TaskStartedTime: timestamppb.New(time.Unix(104, 0)), + }, PreviousCompletedTask: &streamservice.WorkflowTaskEpoch{StartedEventId: 2}, RunId: "run", }, read.GetRecords()[2].GetSupersession()) diff --git a/chasm/lib/stream/tasks.go b/chasm/lib/stream/tasks.go index a2399065004..268420ea442 100644 --- a/chasm/lib/stream/tasks.go +++ b/chasm/lib/stream/tasks.go @@ -20,7 +20,8 @@ func (h *timerTaskHandler) Validate( _ chasm.TaskInvocation, task *streampb.StreamTimerTask, ) (bool, error) { - return s.GetTimerDeadline() != nil && s.GetTimerDeadline().AsTime().Equal(task.GetDeadline().AsTime()), nil + return s.GetTimerDeadline() != nil && + s.GetTimerDeadline().AsTime().Equal(task.GetDeadline().AsTime()), nil } func (h *timerTaskHandler) Execute( diff --git a/chasm/lib/stream/vectors_test.go b/chasm/lib/stream/vectors_test.go index 81b26f4a541..610fe747b0c 100644 --- a/chasm/lib/stream/vectors_test.go +++ b/chasm/lib/stream/vectors_test.go @@ -151,8 +151,14 @@ func TestCompareEpochsMatchesTheCrateKeys(t *testing.T) { require.NotEmpty(t, vectors) for _, a := range vectors { for _, b := range vectors { - require.Equal(t, strings.Compare(a.Key, b.Key), CompareEpochs(a.Epoch.proto(), b.Epoch.proto()), - "%s against %s", a.Key, b.Key) + require.Equal( + t, + strings.Compare(a.Key, b.Key), + CompareEpochs(a.Epoch.proto(), b.Epoch.proto()), + "%s against %s", + a.Key, + b.Key, + ) } } } diff --git a/service/frontend/fx.go b/service/frontend/fx.go index c7aaa515b63..fe479fca24a 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -437,7 +437,8 @@ func RedirectionInterceptorProvider( timeSource, clusterMetadata, ).WithRedirectResponses(chasmstream.RedirectableMethods()). - WithAdditionalAllowedMethods(slices.Collect(maps.Keys(chasmstream.RedirectableMethods()))...) + WithAdditionalAllowedMethods( + slices.Collect(maps.Keys(chasmstream.RedirectableMethods()))...) } func BusinessIDInterceptorProvider( diff --git a/service/frontend/nexus_direct.go b/service/frontend/nexus_direct.go index bf559cc31d2..2257376d76a 100644 --- a/service/frontend/nexus_direct.go +++ b/service/frontend/nexus_direct.go @@ -27,7 +27,9 @@ func (wh *WorkflowHandler) StartNexusOperation( return nil, serviceerror.NewUnimplemented("Direct Nexus calls are disabled") } if req.GetEndpoint() != commonnexus.SystemEndpoint { - return nil, serviceerror.NewUnimplemented("Direct Nexus calls serve only the __temporal_system endpoint") + return nil, serviceerror.NewUnimplemented( + "Direct Nexus calls serve only the __temporal_system endpoint", + ) } ns, err := wh.namespaceRegistry.GetNamespace(namespace.Name(req.GetNamespace())) if err != nil { @@ -37,25 +39,33 @@ func (wh *WorkflowHandler) StartNexusOperation( if requestID == "" { requestID = uuid.NewString() } - processed, err := wh.registry.NexusEndpointProcessor.ProcessInput(chasm.NexusOperationProcessorContext{ - Namespace: ns, - RequestID: requestID, - ReserializeInputPayload: true, - }, req.GetService(), req.GetOperation(), req.GetInput()) + processed, err := wh.registry.NexusEndpointProcessor.ProcessInput( + chasm.NexusOperationProcessorContext{ + Namespace: ns, + RequestID: requestID, + ReserializeInputPayload: true, + }, + req.GetService(), + req.GetOperation(), + req.GetInput(), + ) if err != nil { return nil, err } - resp, err := wh.historyClient.StartNexusOperation(ctx, &historyservice.StartNexusOperationRequest{ - NamespaceId: ns.ID().String(), - ShardId: processed.RoutingKey.ShardID(wh.config.NumHistoryShards), - Request: &nexuspb.StartOperationRequest{ - Service: req.GetService(), - Operation: req.GetOperation(), - Payload: processed.ReserializedInputPayload, - RequestId: requestID, - Links: workflowEventLinks(req.GetLinks(), requestID), + resp, err := wh.historyClient.StartNexusOperation( + ctx, + &historyservice.StartNexusOperationRequest{ + NamespaceId: ns.ID().String(), + ShardId: processed.RoutingKey.ShardID(wh.config.NumHistoryShards), + Request: &nexuspb.StartOperationRequest{ + Service: req.GetService(), + Operation: req.GetOperation(), + Payload: processed.ReserializedInputPayload, + RequestId: requestID, + Links: workflowEventLinks(req.GetLinks(), requestID), + }, }, - }) + ) if err != nil { return nil, err } @@ -63,7 +73,9 @@ func (wh *WorkflowHandler) StartNexusOperation( case *nexuspb.StartOperationResponse_SyncSuccess: return &workflowservice.StartNexusOperationResponse{ Variant: &workflowservice.StartNexusOperationResponse_SyncSuccess_{ - SyncSuccess: &workflowservice.StartNexusOperationResponse_SyncSuccess{Payload: v.SyncSuccess.GetPayload()}, + SyncSuccess: &workflowservice.StartNexusOperationResponse_SyncSuccess{ + Payload: v.SyncSuccess.GetPayload(), + }, }, Links: v.SyncSuccess.GetLinks(), }, nil @@ -79,7 +91,9 @@ func (wh *WorkflowHandler) StartNexusOperation( case *nexuspb.StartOperationResponse_Failure: return &workflowservice.StartNexusOperationResponse{ Variant: &workflowservice.StartNexusOperationResponse_OperationFailure_{ - OperationFailure: &workflowservice.StartNexusOperationResponse_OperationFailure{Failure: v.Failure}, + OperationFailure: &workflowservice.StartNexusOperationResponse_OperationFailure{ + Failure: v.Failure, + }, }, }, nil default: diff --git a/tests/stream_test.go b/tests/stream_test.go index 3ae2e3e7f1a..9ed123f48c0 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -65,22 +65,28 @@ type streamCaller struct { func (s *StreamTestSuite) newCaller(env *testcore.TestEnv) *streamCaller { c := &streamCaller{s: s, env: env, taskQueue: testcore.RandomizeStr(s.T().Name())} - run, err := env.SdkClient().ExecuteWorkflow(s.Context(), client.StartWorkflowOptions{TaskQueue: c.taskQueue}, "stream-caller") + options := client.StartWorkflowOptions{TaskQueue: c.taskQueue} + run, err := env.SdkClient().ExecuteWorkflow(s.Context(), options, "stream-caller") s.NoError(err) c.run = run s.T().Cleanup(func() { - _ = env.SdkClient().TerminateWorkflow(s.Context(), run.GetID(), run.GetRunID(), "test cleanup") + _ = env.SdkClient(). + TerminateWorkflow(s.Context(), run.GetID(), run.GetRunID(), "test cleanup") }) c.poll() return c } func (c *streamCaller) poll() { - resp, err := c.env.FrontendClient().PollWorkflowTaskQueue(c.s.Context(), &workflowservice.PollWorkflowTaskQueueRequest{ - Namespace: c.env.Namespace().String(), - TaskQueue: &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - Identity: "test", - }) + resp, err := c.env.FrontendClient(). + PollWorkflowTaskQueue(c.s.Context(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: c.env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{ + Name: c.taskQueue, + Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + }, + Identity: "test", + }) c.s.NoError(err) c.s.NotEmpty(resp.GetTaskToken()) c.token = resp.GetTaskToken() @@ -107,31 +113,67 @@ func (c *streamCaller) schedule(ops ...streamOp) []int64 { return ids } +// completeTask completes the open Workflow Task with commands. It polls no next task, since the +// commands may end the run. +func (c *streamCaller) completeTask(commands ...*commandpb.Command) error { + _, err := c.env.FrontendClient().RespondWorkflowTaskCompleted(c.s.Context(), + &workflowservice.RespondWorkflowTaskCompletedRequest{ + Identity: "test", + TaskToken: c.token, + Commands: commands, + }) + return err +} + func (c *streamCaller) respond(commands ...*commandpb.Command) { - _, err := c.env.FrontendClient().RespondWorkflowTaskCompleted(c.s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Identity: "test", - TaskToken: c.token, - Commands: commands, - }) - c.s.NoError(err) + c.s.NoError(c.completeTask(commands...)) c.poll() } +// continueAsNew continues the run as new on the caller's task queue. +func (c *streamCaller) continueAsNew() *commandpb.Command { + taskQueue := &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + attributes := &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, + TaskQueue: taskQueue, + } + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ + ContinueAsNewWorkflowExecutionCommandAttributes: attributes, + }, + } +} + +// completeCommand completes the run with result. +func completeCommand(result *commonpb.Payloads) *commandpb.Command { + attributes := &commandpb.CompleteWorkflowExecutionCommandAttributes{Result: result} + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: attributes, + }, + } +} + // outcome waits for the end of the operation that scheduledID scheduled. It answers whether the // operation started async first. func (c *streamCaller) outcome(scheduledID int64, result proto.Message) (*failurepb.Failure, bool) { for { started := false for _, event := range c.history { - if attrs := event.GetNexusOperationStartedEventAttributes(); attrs != nil && attrs.GetScheduledEventId() == scheduledID { + if attrs := event.GetNexusOperationStartedEventAttributes(); attrs != nil && + attrs.GetScheduledEventId() == scheduledID { started = true } - if attrs := event.GetNexusOperationCompletedEventAttributes(); attrs != nil && attrs.GetScheduledEventId() == scheduledID { + if attrs := event.GetNexusOperationCompletedEventAttributes(); attrs != nil && + attrs.GetScheduledEventId() == scheduledID { c.s.NoError(sdkconverter.PreferProtoDataConverter.FromPayloads( &commonpb.Payloads{Payloads: []*commonpb.Payload{attrs.GetResult()}}, result)) return nil, started } - if attrs := event.GetNexusOperationFailedEventAttributes(); attrs != nil && attrs.GetScheduledEventId() == scheduledID { + if attrs := event.GetNexusOperationFailedEventAttributes(); attrs != nil && + attrs.GetScheduledEventId() == scheduledID { return attrs.GetFailure(), started } } @@ -151,15 +193,16 @@ type streamOp struct { // nexusCommand schedules op as a System Nexus operation. func nexusCommand(op streamOp) *commandpb.Command { + attributes := &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: op.name, + Input: payloads.MustEncodeSingle(op.request), + } return &commandpb.Command{ CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ - ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ - Endpoint: commonnexus.SystemEndpoint, - Service: stream.ServiceName, - Operation: op.name, - Input: payloads.MustEncodeSingle(op.request), - }, + ScheduleNexusOperationCommandAttributes: attributes, }, } } @@ -171,7 +214,10 @@ func ref(id string) *streamservice.StreamRef { func records(topics ...string) []*streamservice.AppendRecord { out := make([]*streamservice.AppendRecord, 0, len(topics)) for _, topic := range topics { - out = append(out, &streamservice.AppendRecord{Topic: topic, Body: payload.EncodeString(topic)}) + out = append( + out, + &streamservice.AppendRecord{Topic: topic, Body: payload.EncodeString(topic)}, + ) } return out } @@ -223,24 +269,52 @@ func (s *StreamTestSuite) TestWorkflowReadAndDescribe() { _, err := sc.CreateStream(ctx, &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) s.NoError(err) appended, err := sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: writer, Sequence: 1, Records: records("turns", "tools"), + Namespace: ns, Stream: ref(id), Producer: writer, Sequence: 1, + Records: records("turns", "tools"), }) s.NoError(err) var page streamservice.ReadRecordsResponse - failure, started := c.outcome(c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id)}})[0], &page) + failure, started := c.outcome( + c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id)}})[0], + &page, + ) s.Nil(failure) s.False(started) s.Equal([]string{"turns", "tools"}, kindsAndTopics(&page)) var described streamservice.DescribeStreamResponse - s.Nil(c.call(streamOp{"DescribeStream", &streamservice.DescribeStreamRequest{Stream: ref(id)}}, &described)) + s.Nil( + c.call( + streamOp{"DescribeStream", &streamservice.DescribeStreamRequest{Stream: ref(id)}}, + &described, + ), + ) s.Equal(appended.GetLastOffset(), described.GetLastOffset()) - _, err = sc.CloseStream(ctx, &streamservice.CloseStreamRequest{Namespace: ns, Stream: ref(id), Producer: clientProducer(), Result: payload.EncodeString("done")}) + _, err = sc.CloseStream( + ctx, + &streamservice.CloseStreamRequest{ + Namespace: ns, + Stream: ref(id), + Producer: clientProducer(), + Result: payload.EncodeString("done"), + }, + ) s.NoError(err) var last streamservice.ReadRecordsResponse - s.Nil(c.call(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(id), FromOffset: page.GetNextOffset()}}, &last)) + s.Nil( + c.call( + streamOp{ + "ReadRecords", + &streamservice.ReadRecordsRequest{ + Stream: ref(id), + FromOffset: page.GetNextOffset(), + }, + }, + &last, + ), + ) s.Equal([]string{"close"}, kindsAndTopics(&last)) s.True(last.GetDone()) } @@ -252,7 +326,10 @@ func (s *StreamTestSuite) TestWorkflowWakeWaitsForAnAppend() { c := s.newCaller(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ @@ -260,7 +337,8 @@ func (s *StreamTestSuite) TestWorkflowWakeWaitsForAnAppend() { }})[0] for i, topic := range []string{"other", "turns"} { _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: int64(i + 1), Records: records(topic), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: int64(i + 1), + Records: records(topic), }) s.NoError(err) } @@ -279,13 +357,23 @@ func (s *StreamTestSuite) TestWorkflowWakeEndsOnClose() { c := s.newCaller(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) read := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ Stream: ref(id), WakeOnly: true, }})[0] - _, err = sc.CloseStream(s.Context(), &streamservice.CloseStreamRequest{Namespace: ns, Stream: ref(id), Producer: clientProducer()}) + _, err = sc.CloseStream( + s.Context(), + &streamservice.CloseStreamRequest{ + Namespace: ns, + Stream: ref(id), + Producer: clientProducer(), + }, + ) s.NoError(err) var page streamservice.ReadRecordsResponse @@ -299,13 +387,21 @@ func (s *StreamTestSuite) TestWorkflowWakeEndsOnClose() { func (s *StreamTestSuite) TestWorkflowReadOfAMissingStreamFails() { env := s.newEnv() c := s.newCaller(env) - failure := c.call(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: ref(testcore.RandomizeStr("missing"))}}, - &streamservice.ReadRecordsResponse{}) + failure := c.call( + streamOp{ + "ReadRecords", + &streamservice.ReadRecordsRequest{Stream: ref(testcore.RandomizeStr("missing"))}, + }, + &streamservice.ReadRecordsResponse{}, + ) s.Equal(stream.ReasonNotFound, causeType(failure)) } func (s *StreamTestSuite) streamClient(env *testcore.TestEnv) streamservice.StreamServiceClient { - conn, err := grpc.NewClient(env.FrontendGRPCAddress(), grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient( + env.FrontendGRPCAddress(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) s.NoError(err) s.T().Cleanup(func() { _ = conn.Close() }) return streamservice.NewStreamServiceClient(conn) @@ -326,7 +422,10 @@ func (s *StreamTestSuite) TestClientCreateAppendReadClose() { ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - created, err := sc.CreateStream(ctx, &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + created, err := sc.CreateStream( + ctx, + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) s.True(created.GetCreated()) generated, err := sc.CreateStream(ctx, &streamservice.CreateStreamRequest{Namespace: ns}) @@ -334,34 +433,61 @@ func (s *StreamTestSuite) TestClientCreateAppendReadClose() { s.NotEmpty(generated.GetStream().GetId()) appended, err := sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, Records: records("turns", "tool/search"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("turns", "tool/search"), }) s.NoError(err) resent, err := sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, Records: records("turns", "tool/search"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("turns", "tool/search"), }) s.NoError(err) s.Equal(appended.GetFirstOffset(), resent.GetFirstOffset()) - page, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id), TopicPrefixes: []string{"tool/"}}) + page, err := sc.ReadRecords( + ctx, + &streamservice.ReadRecordsRequest{ + Namespace: ns, + Stream: ref(id), + TopicPrefixes: []string{"tool/"}, + }, + ) s.NoError(err) s.Equal([]string{"tool/search"}, kindsAndTopics(page)) s.Equal(appended.GetLastOffset()+1, page.GetNextOffset()) - described, err := sc.DescribeStream(ctx, &streamservice.DescribeStreamRequest{Namespace: ns, Stream: ref(id)}) + described, err := sc.DescribeStream( + ctx, + &streamservice.DescribeStreamRequest{Namespace: ns, Stream: ref(id)}, + ) s.NoError(err) s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) s.Equal(appended.GetLastOffset(), described.GetLastOffset()) - _, err = sc.CloseStream(ctx, &streamservice.CloseStreamRequest{Namespace: ns, Stream: ref(id), Producer: clientProducer()}) + _, err = sc.CloseStream( + ctx, + &streamservice.CloseStreamRequest{ + Namespace: ns, + Stream: ref(id), + Producer: clientProducer(), + }, + ) s.NoError(err) - last, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id), FromOffset: page.GetNextOffset()}) + last, err := sc.ReadRecords( + ctx, + &streamservice.ReadRecordsRequest{ + Namespace: ns, + Stream: ref(id), + FromOffset: page.GetNextOffset(), + }, + ) s.NoError(err) s.Equal([]string{"close"}, kindsAndTopics(last)) s.True(last.GetDone()) _, err = sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 2, Records: records("late"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 2, + Records: records("late"), }) st := status.Convert(err) s.Equal(codes.FailedPrecondition, st.Code()) @@ -385,7 +511,8 @@ func (s *StreamTestSuite) TestClientReadWaitsForAnAppend() { done := make(chan result, 1) go func() { resp, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{ - Namespace: ns, Stream: ref(id), Start: streamservice.READ_START_END, Wait: durationpb.New(30 * time.Second), + Namespace: ns, Stream: ref(id), Start: streamservice.READ_START_END, + Wait: durationpb.New(30 * time.Second), }) done <- result{resp, err} }() @@ -394,7 +521,8 @@ func (s *StreamTestSuite) TestClientReadWaitsForAnAppend() { s.Await(func(s *StreamTestSuite) { sequence++ _, err := sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: sequence, Records: records("turns"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: sequence, + Records: records("turns"), }) s.NoError(err) select { @@ -441,10 +569,14 @@ func (s *StreamTestSuite) TestWorkflowWakeWaitsForTheCreate() { WakeOnly: true, CreateWait: durationpb.New(30 * time.Second), }})[0] - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, Records: records("turns"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("turns"), }) s.NoError(err) @@ -493,20 +625,36 @@ func (s *StreamTestSuite) TestAbandonedWorkflowStreamCanStillBeCreated() { } // closeWithRun creates a Stream that closes with the caller's run chain. -func (s *StreamTestSuite) closeWithRun(sc streamservice.StreamServiceClient, env *testcore.TestEnv, c *streamCaller) *streamservice.StreamRef { +func (s *StreamTestSuite) closeWithRun( + sc streamservice.StreamServiceClient, + env *testcore.TestEnv, + c *streamCaller, +) *streamservice.StreamRef { id := testcore.RandomizeStr("chat") _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ - Namespace: env.Namespace().String(), - StreamId: id, - Owner: workflowWriter(c.run.GetRunID(), 3), - Ref: &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID()}, + Namespace: env.Namespace().String(), + StreamId: id, + Owner: workflowWriter(c.run.GetRunID(), 3), + Ref: &streamservice.StreamRef{ + Id: id, + WorkflowId: c.run.GetID(), + FirstExecutionRunId: c.run.GetRunID(), + }, CloseWithRun: true, }) s.NoError(err) - return &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID()} + return &streamservice.StreamRef{ + Id: id, + WorkflowId: c.run.GetID(), + FirstExecutionRunId: c.run.GetRunID(), + } } -func (s *StreamTestSuite) readClose(sc streamservice.StreamServiceClient, env *testcore.TestEnv, owned *streamservice.StreamRef) *streamservice.StreamRecord { +func (s *StreamTestSuite) readClose( + sc streamservice.StreamServiceClient, + env *testcore.TestEnv, + owned *streamservice.StreamRef, +) *streamservice.StreamRecord { page, err := sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{ Namespace: env.Namespace().String(), Stream: owned, Wait: durationpb.New(20 * time.Second), }) @@ -521,19 +669,7 @@ func (s *StreamTestSuite) TestStreamClosesWithTheOwnerResult() { c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) - _, err := env.FrontendClient().RespondWorkflowTaskCompleted(s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Identity: "test", - TaskToken: c.token, - Commands: []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{ - Result: payloads.EncodeString("answer"), - }, - }, - }}, - }) - s.NoError(err) + s.NoError(c.completeTask(completeCommand(payloads.EncodeString("answer")))) closed := s.readClose(sc, env, owned) s.Equal(payload.EncodeString("answer").GetData(), closed.GetCloseResult().GetData()) @@ -546,7 +682,9 @@ func (s *StreamTestSuite) TestStreamClosesWithATerminatedOwner() { c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) - s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), c.run.GetRunID(), "stop")) + s.NoError( + env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), c.run.GetRunID(), "stop"), + ) closed := s.readClose(sc, env, owned) s.NotNil(closed.GetCloseFailure().GetTerminatedFailureInfo()) @@ -555,7 +693,10 @@ func (s *StreamTestSuite) TestStreamClosesWithATerminatedOwner() { // closeOp closes a Workflow Stream through System Nexus, as the calling run. func closeOp(owned *streamservice.StreamRef, result string) streamOp { - return streamOp{"CloseStream", &streamservice.CloseStreamRequest{Stream: owned, Result: payload.EncodeString(result)}} + return streamOp{ + "CloseStream", + &streamservice.CloseStreamRequest{Stream: owned, Result: payload.EncodeString(result)}, + } } func (s *StreamTestSuite) TestWorkflowCloseCommandClosesItsStream() { @@ -580,7 +721,10 @@ func (s *StreamTestSuite) TestCloseCommandOfAnotherChainIsRefused() { failure := s.newCaller(env).call(closeOp(owned, "answer"), &streamservice.CloseStreamResponse{}) s.Equal(stream.ReasonNotOwner, causeType(failure)) - described, err := sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{Namespace: env.Namespace().String(), Stream: owned}) + described, err := sc.DescribeStream( + s.Context(), + &streamservice.DescribeStreamRequest{Namespace: env.Namespace().String(), Stream: owned}, + ) s.NoError(err) s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) } @@ -594,18 +738,22 @@ func (s *StreamTestSuite) TestDirectCloseCallIsRefused() { c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) - resp, err := env.FrontendClient().StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ - Namespace: ns, - Endpoint: commonnexus.SystemEndpoint, - Service: stream.ServiceName, - Operation: "CloseStream", - Input: payloads.MustEncodeSingle(closeOp(owned, "answer").request), - Links: []*commonpb.Link{readerLink(env, c.run)}, - }) + resp, err := env.FrontendClient(). + StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ + Namespace: ns, + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: "CloseStream", + Input: payloads.MustEncodeSingle(closeOp(owned, "answer").request), + Links: []*commonpb.Link{readerLink(env, c.run)}, + }) s.NoError(err) s.Equal(stream.ReasonInvalid, causeType(resp.GetOperationFailure().GetFailure())) - described, err := sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{Namespace: ns, Stream: owned}) + described, err := sc.DescribeStream( + s.Context(), + &streamservice.DescribeStreamRequest{Namespace: ns, Stream: owned}, + ) s.NoError(err) s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) } @@ -618,27 +766,28 @@ func (s *StreamTestSuite) TestCloseCommandOfAFailedWorkflowTaskIsDropped() { c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) - _, err := env.FrontendClient().RespondWorkflowTaskCompleted(s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Identity: "test", - TaskToken: c.token, - Commands: []*commandpb.Command{ - nexusCommand(closeOp(owned, "failed task")), - { - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ - ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{}, - }, - }, + // An Activity with empty attributes is a bad command. + empty := &commandpb.ScheduleActivityTaskCommandAttributes{} + bad := &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: empty, }, - }) - s.Error(err) + } + s.Error(c.completeTask(nexusCommand(closeOp(owned, "failed task")), bad)) c.poll() s.Nil(c.call(closeOp(owned, "retried task"), &streamservice.CloseStreamResponse{})) - page, err := sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: owned}) + page, err := sc.ReadRecords( + s.Context(), + &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: owned}, + ) s.NoError(err) s.Equal([]string{"close"}, kindsAndTopics(page)) - s.Equal(payload.EncodeString("retried task").GetData(), page.GetRecords()[0].GetCloseResult().GetData()) + s.Equal( + payload.EncodeString("retried task").GetData(), + page.GetRecords()[0].GetCloseResult().GetData(), + ) } // A cron run's end isn't the chain's end, so the next cron run writes on. The terminate of a later @@ -659,13 +808,7 @@ func (s *StreamTestSuite) TestCronRunEndKeepsTheStreamOpen() { owned := s.closeWithRun(sc, env, c) // Run 1 completes and the next cron run starts. - c.respond(&commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb. - CompleteWorkflowExecutionCommandAttributes{}, - }, - }) + c.respond(completeCommand(nil)) appended, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ Namespace: env.Namespace().String(), Stream: owned, Producer: workflowWriter(c.current, 3), Sequence: 1, Records: records("next"), @@ -700,13 +843,17 @@ func (s *StreamTestSuite) TestDeferredWorkflowBatchIsWithdrawnByAFailedTask() { }) s.NoError(err) _, err = sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: workflowWriter("run-1", 7), Sequence: 1, Records: records("b"), + Namespace: ns, Stream: ref(id), Producer: workflowWriter("run-1", 7), Sequence: 1, + Records: records("b"), PreviousCompletedTask: &streamservice.WorkflowTaskEpoch{StartedEventId: 2, Attempt: 1}, FailedTasks: []int64{3}, }) s.NoError(err) - page, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id)}) + page, err := sc.ReadRecords( + ctx, + &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id)}, + ) s.NoError(err) s.Equal([]string{"a", "", "b"}, kindsAndTopics(page)) s.Equal(streamservice.STREAM_RECORD_KIND_SUPERSEDED, page.GetRecords()[1].GetKind()) @@ -721,17 +868,24 @@ func (s *StreamTestSuite) TestWorkflowReadThatIsNotAWakeAnswersAtOnce() { c := s.newCaller(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) appended, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, Records: records("other"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("other"), }) s.NoError(err) var page streamservice.ReadRecordsResponse - failure, started := c.outcome(c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ - Stream: ref(id), Topics: []string{"turns"}, Wait: durationpb.New(time.Minute), - }})[0], &page) + failure, started := c.outcome( + c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: ref(id), Topics: []string{"turns"}, Wait: durationpb.New(time.Minute), + }})[0], + &page, + ) s.Nil(failure) s.False(started) s.Empty(page.GetRecords()) @@ -747,7 +901,10 @@ func (s *StreamTestSuite) TestLargeAppendWithManyReadersStaysUnderTheStateLimit( c := s.newCaller(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) ops := make([]streamOp, 0, 5) @@ -796,7 +953,13 @@ func (s *StreamTestSuite) TestNestedStreamWakesItsWorkflow() { s.Empty(page.GetRecords()) // The Stream id names no execution of its own. - _, err = sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{Namespace: env.Namespace().String(), Stream: ref(owned.GetId())}) + _, err = sc.DescribeStream( + s.Context(), + &streamservice.DescribeStreamRequest{ + Namespace: env.Namespace().String(), + Stream: ref(owned.GetId()), + }, + ) s.Equal(codes.NotFound, status.Code(err)) } @@ -807,15 +970,7 @@ func (s *StreamTestSuite) TestStreamClosesWithTheLastRunOfTheChain() { owned := s.closeWithRun(sc, env, c) // Run 1 continues as new. The Stream stays open in run 1's tree. - c.respond(&commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ - ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, - TaskQueue: &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - }, - }, - }) + c.respond(c.continueAsNew()) described, err := sc.DescribeStream(s.Context(), &streamservice.DescribeStreamRequest{ Namespace: env.Namespace().String(), Stream: owned, }) @@ -823,19 +978,7 @@ func (s *StreamTestSuite) TestStreamClosesWithTheLastRunOfTheChain() { s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) // Run 2 completes, which ends the chain. - _, err = env.FrontendClient().RespondWorkflowTaskCompleted(s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ - Identity: "test", - TaskToken: c.token, - Commands: []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{ - Result: payloads.EncodeString("run-2"), - }, - }, - }}, - }) - s.NoError(err) + s.NoError(c.completeTask(completeCommand(payloads.EncodeString("run-2")))) closed := s.readClose(sc, env, owned) s.Equal(payload.EncodeString("run-2").GetData(), closed.GetCloseResult().GetData()) s.Equal(streamservice.CLOSE_STATUS_COMPLETED, closed.GetCloseStatus()) @@ -850,31 +993,24 @@ func (s *StreamTestSuite) TestResetToAnEarlierRunKeepsTheStreamOpen() { ns := env.Namespace().String() owned := s.closeWithRun(sc, env, c) first := c.run.GetRunID() - c.respond(&commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ - ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb. - ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, - TaskQueue: &taskqueuepb.TaskQueue{ - Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL, - }, - }, - }, - }) + c.respond(c.continueAsNew()) s.NotEqual(first, c.current) reset, err := env.FrontendClient().ResetWorkflowExecution(s.Context(), &workflowservice.ResetWorkflowExecutionRequest{ - Namespace: ns, - WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: c.run.GetID(), RunId: first}, + Namespace: ns, + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: c.run.GetID(), + RunId: first, + }, Reason: "test", WorkflowTaskFinishEventId: 4, RequestId: uuid.NewString(), }) s.NoError(err) s.T().Cleanup(func() { - _ = env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), reset.GetRunId(), "test cleanup") + _ = env.SdkClient(). + TerminateWorkflow(s.Context(), c.run.GetID(), reset.GetRunId(), "test cleanup") }) // The chain's end would close the Stream through a task on the terminated run. @@ -885,7 +1021,9 @@ func (s *StreamTestSuite) TestResetToAnEarlierRunKeepsTheStreamOpen() { s.Equal(streamservice.STREAM_STATUS_OPEN, described.GetStatus()) // The reset run's end is the chain's end. - s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), reset.GetRunId(), "stop")) + s.NoError( + env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), reset.GetRunId(), "stop"), + ) s.Equal(streamservice.CLOSE_STATUS_TERMINATED, s.readClose(sc, env, owned).GetCloseStatus()) } @@ -895,18 +1033,7 @@ func (s *StreamTestSuite) TestChainEndLeavesAFirstRunWithNoStreamAlone() { c := s.newCaller(env) ns := env.Namespace().String() first := c.run.GetRunID() - c.respond(&commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ - ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb. - ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, - TaskQueue: &taskqueuepb.TaskQueue{ - Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL, - }, - }, - }, - }) + c.respond(c.continueAsNew()) transitions := func() int64 { resp, err := env.AdminClient().DescribeMutableState(s.Context(), &adminservice.DescribeMutableStateRequest{ @@ -931,29 +1058,38 @@ func (s *StreamTestSuite) directRead( req *streamservice.ReadRecordsRequest, links ...*commonpb.Link, ) (*streamservice.ReadRecordsResponse, *failurepb.Failure) { - resp, err := env.FrontendClient().StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ - Namespace: env.Namespace().String(), - Endpoint: commonnexus.SystemEndpoint, - Service: stream.ServiceName, - Operation: "ReadRecords", - Input: payloads.MustEncodeSingle(req), - Links: links, - }) + resp, err := env.FrontendClient(). + StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ + Namespace: env.Namespace().String(), + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: "ReadRecords", + Input: payloads.MustEncodeSingle(req), + Links: links, + }) s.NoError(err) if failure := resp.GetOperationFailure().GetFailure(); failure != nil { return nil, failure } var page streamservice.ReadRecordsResponse s.NoError(sdkconverter.PreferProtoDataConverter.FromPayloads( - &commonpb.Payloads{Payloads: []*commonpb.Payload{resp.GetSyncSuccess().GetPayload()}}, &page)) + &commonpb.Payloads{ + Payloads: []*commonpb.Payload{resp.GetSyncSuccess().GetPayload()}, + }, + &page, + )) return &page, nil } func (s *StreamTestSuite) historyLength(env *testcore.TestEnv, c *streamCaller) int { - resp, err := env.FrontendClient().GetWorkflowExecutionHistory(s.Context(), &workflowservice.GetWorkflowExecutionHistoryRequest{ - Namespace: env.Namespace().String(), - Execution: &commonpb.WorkflowExecution{WorkflowId: c.run.GetID(), RunId: c.run.GetRunID()}, - }) + resp, err := env.FrontendClient().GetWorkflowExecutionHistory(s.Context(), + &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{ + WorkflowId: c.run.GetID(), + RunId: c.run.GetRunID(), + }, + }) s.NoError(err) return len(resp.GetHistory().GetEvents()) } @@ -969,12 +1105,15 @@ func (s *StreamTestSuite) TestWorkflowReadsDirectlyWithNoRecordsInHistory() { ns := env.Namespace().String() appendTopics := func(sequence int64, topics ...string) { _, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), Sequence: sequence, Records: records(topics...), + Namespace: ns, Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), + Sequence: sequence, Records: records(topics...), }) s.NoError(err) } - wake := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{Stream: owned, WakeOnly: true}})[0] + wake := c.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, + }})[0] appendTopics(1, "a", "b") var woke streamservice.ReadRecordsResponse failure, started := c.outcome(wake, &woke) @@ -984,25 +1123,35 @@ func (s *StreamTestSuite) TestWorkflowReadsDirectlyWithNoRecordsInHistory() { s.Equal(int64(0), woke.GetNextOffset()) before := s.historyLength(env, c) - page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: woke.GetNextOffset()}) + page, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: woke.GetNextOffset()}, + ) s.Nil(failure) s.Equal([]string{"a", "b"}, kindsAndTopics(page)) s.Equal(before, s.historyLength(env, c)) appendTopics(2, "c") - again, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, ToOffset: page.GetNextOffset()}) + again, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, ToOffset: page.GetNextOffset()}, + ) s.Nil(failure) s.Equal(page.GetRecords(), again.GetRecords()) s.Equal(page.GetNextOffset(), again.GetNextOffset()) // A direct read with wait long-polls, as the frontend read does. waited, failure := s.directRead(env, &streamservice.ReadRecordsRequest{ - Stream: owned, Start: streamservice.READ_START_END, Wait: durationpb.New(500 * time.Millisecond), + Stream: owned, Start: streamservice.READ_START_END, + Wait: durationpb.New(500 * time.Millisecond), }) s.Nil(failure) s.Empty(waited.GetRecords()) - _, failure = s.directRead(env, &streamservice.ReadRecordsRequest{Stream: ref(testcore.RandomizeStr("missing"))}) + _, failure = s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: ref(testcore.RandomizeStr("missing"))}, + ) s.Equal(stream.ReasonNotFound, causeType(failure)) } @@ -1016,10 +1165,13 @@ func (s *StreamTestSuite) TestChainOutlivesTheRetentionOfItsFirstRun() { testcore.WithDynamicConfig(dynamicconfig.NamespaceMinRetentionLocal, time.Duration(0)), testcore.WithDynamicConfig(dynamicconfig.RetentionTimerJitterDuration, time.Duration(0)), ) - _, err := env.FrontendClient().UpdateNamespace(s.Context(), &workflowservice.UpdateNamespaceRequest{ - Namespace: env.Namespace().String(), - Config: &namespacepb.NamespaceConfig{WorkflowExecutionRetentionTtl: durationpb.New(time.Second)}, - }) + _, err := env.FrontendClient(). + UpdateNamespace(s.Context(), &workflowservice.UpdateNamespaceRequest{ + Namespace: env.Namespace().String(), + Config: &namespacepb.NamespaceConfig{ + WorkflowExecutionRetentionTtl: durationpb.New(time.Second), + }, + }) s.NoError(err) // Wait for the namespace cache to see the new retention. <-time.After(2 * testcore.NamespaceCacheRefreshInterval) @@ -1027,15 +1179,7 @@ func (s *StreamTestSuite) TestChainOutlivesTheRetentionOfItsFirstRun() { sc := s.streamClient(env) c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) - c.respond(&commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ - ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ - WorkflowType: &commonpb.WorkflowType{Name: "stream-caller"}, - TaskQueue: &taskqueuepb.TaskQueue{Name: c.taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - }, - }, - }) + c.respond(c.continueAsNew()) // Several retentions of the first run pass while run 2 is open. <-time.After(4 * time.Second) @@ -1044,7 +1188,10 @@ func (s *StreamTestSuite) TestChainOutlivesTheRetentionOfItsFirstRun() { Producer: workflowWriter(c.current, 3), Sequence: 1, Records: records("late"), }) s.NoError(err) - page, err := sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: owned}) + page, err := sc.ReadRecords( + s.Context(), + &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: owned}, + ) s.NoError(err) s.Equal([]string{"late"}, kindsAndTopics(page)) } @@ -1063,25 +1210,42 @@ func (s *StreamTestSuite) TestActivityStreamOutlivesTheRetentionOfItsActivity() ns := env.Namespace().String() _, err := env.FrontendClient().UpdateNamespace(ctx, &workflowservice.UpdateNamespaceRequest{ Namespace: ns, - Config: &namespacepb.NamespaceConfig{WorkflowExecutionRetentionTtl: durationpb.New(time.Second)}, + Config: &namespacepb.NamespaceConfig{ + WorkflowExecutionRetentionTtl: durationpb.New(time.Second), + }, }) s.NoError(err) // Wait for the namespace cache to see the new retention. <-time.After(2 * testcore.NamespaceCacheRefreshInterval) activityID := "tool-" + uuid.NewString() - taskQueue := &taskqueuepb.TaskQueue{Name: "tools-" + uuid.NewString(), Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + taskQueue := &taskqueuepb.TaskQueue{ + Name: "tools-" + uuid.NewString(), + Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + } started, err := env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ - Namespace: ns, ActivityId: activityID, ActivityType: &commonpb.ActivityType{Name: "tool"}, - Identity: "test", TaskQueue: taskQueue, StartToCloseTimeout: durationpb.New(time.Minute), + Namespace: ns, + ActivityId: activityID, + ActivityType: &commonpb.ActivityType{Name: "tool"}, + Identity: "test", + TaskQueue: taskQueue, + StartToCloseTimeout: durationpb.New(time.Minute), }) s.NoError(err) task, err := env.FrontendClient().PollActivityTaskQueue(ctx, - &workflowservice.PollActivityTaskQueueRequest{Namespace: ns, TaskQueue: taskQueue, Identity: "test"}) + &workflowservice.PollActivityTaskQueueRequest{ + Namespace: ns, + TaskQueue: taskQueue, + Identity: "test", + }) s.NoError(err) id := testcore.RandomizeStr("tokens") - owned := &streamservice.StreamRef{Id: id, ActivityId: activityID, ActivityRunId: started.GetRunId()} + owned := &streamservice.StreamRef{ + Id: id, + ActivityId: activityID, + ActivityRunId: started.GetRunId(), + } owner := &streamservice.Producer{ Kind: streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY, Id: "sa:" + activityID + ":" + started.GetRunId(), @@ -1107,7 +1271,10 @@ func (s *StreamTestSuite) TestActivityStreamOutlivesTheRetentionOfItsActivity() // Several namespace retentions pass after the Activity's end. <-time.After(4 * time.Second) - page, err := sc.ReadRecords(ctx, &streamservice.ReadRecordsRequest{Namespace: ns, Stream: owned}) + page, err := sc.ReadRecords( + ctx, + &streamservice.ReadRecordsRequest{Namespace: ns, Stream: owned}, + ) s.NoError(err) s.Equal([]string{"tokens", "close"}, kindsAndTopics(page)) } @@ -1148,7 +1315,9 @@ func (s *StreamTestSuite) TestWorkflowStreamWithNoReaderTrimsPastTheCap() { var first int64 for sequence := int64(1); sequence <= 5; sequence++ { resp, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: env.Namespace().String(), Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), + Namespace: env.Namespace(). + String(), + Stream: owned, Producer: workflowWriter(c.run.GetRunID(), 3), Sequence: sequence, Records: []*streamservice.AppendRecord{ {Topic: "big", Body: &commonpb.Payload{Data: make([]byte, 1024*1024)}}, }, @@ -1162,7 +1331,10 @@ func (s *StreamTestSuite) TestWorkflowStreamWithNoReaderTrimsPastTheCap() { _, err := sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{ Namespace: env.Namespace().String(), Stream: owned, FromOffset: first, }) - s.Equal(stream.ReasonExpired, status.Convert(err).Details()[0].(*streamservice.StreamErrorDetails).GetReason()) + s.Equal( + stream.ReasonExpired, + status.Convert(err).Details()[0].(*streamservice.StreamErrorDetails).GetReason(), + ) } // The range that an open Workflow run read stays past the Retention, so its replay reads the same @@ -1173,35 +1345,51 @@ func (s *StreamTestSuite) TestWorkflowReaderKeepsItsRangePastTheRetention() { c := s.newCaller(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - owned := &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID()} + owned := &streamservice.StreamRef{ + Id: id, + WorkflowId: c.run.GetID(), + FirstExecutionRunId: c.run.GetRunID(), + } _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ Namespace: ns, StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), Owner: workflowWriter(c.run.GetRunID(), 3), }) s.NoError(err) appended, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: owned, Producer: clientProducer(), Sequence: 1, Records: records("a"), + Namespace: ns, Stream: owned, Producer: clientProducer(), Sequence: 1, + Records: records("a"), }) s.NoError(err) readerRun := s.newCaller(env).run - reader := &commonpb.Link{Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: ns, WorkflowId: readerRun.GetID(), RunId: readerRun.GetRunID(), - }}} + reader := &commonpb.Link{ + Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ + Namespace: ns, WorkflowId: readerRun.GetID(), RunId: readerRun.GetRunID(), + }}, + } page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned}, reader) s.Nil(failure) s.Equal([]string{"a"}, kindsAndTopics(page)) // Past the Retention, and past a reader check, the replay reads the same page. time.Sleep(3 * time.Second) //nolint:forbidigo // The Retention and the check run on real time. - again, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, ToOffset: page.GetNextOffset()}) + again, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, ToOffset: page.GetNextOffset()}, + ) s.Nil(failure) s.Equal(page.GetRecords(), again.GetRecords()) s.Equal(page.GetNextOffset(), again.GetNextOffset()) - s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), readerRun.GetID(), readerRun.GetRunID(), "reader done")) + s.NoError( + env.SdkClient(). + TerminateWorkflow(s.Context(), readerRun.GetID(), readerRun.GetRunID(), "reader done"), + ) s.AwaitTrue(func() bool { - _, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: appended.GetFirstOffset()}) + _, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: appended.GetFirstOffset()}, + ) return failure != nil && causeType(failure) == stream.ReasonExpired }, 10*time.Second, 200*time.Millisecond) } @@ -1213,14 +1401,19 @@ func (s *StreamTestSuite) TestReadOfADroppedRangeFailsAsExpired() { sc := s.streamClient(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) var first int64 for sequence := int64(1); sequence <= 3; sequence++ { resp, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: sequence, - Records: []*streamservice.AppendRecord{{Topic: "big", Body: &commonpb.Payload{Data: make([]byte, 1536*1024)}}}, + Records: []*streamservice.AppendRecord{ + {Topic: "big", Body: &commonpb.Payload{Data: make([]byte, 1536*1024)}}, + }, }) s.NoError(err) if first == 0 { @@ -1228,13 +1421,19 @@ func (s *StreamTestSuite) TestReadOfADroppedRangeFailsAsExpired() { } } - _, err = sc.ReadRecords(s.Context(), &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id), FromOffset: first}) + _, err = sc.ReadRecords( + s.Context(), + &streamservice.ReadRecordsRequest{Namespace: ns, Stream: ref(id), FromOffset: first}, + ) st := status.Convert(err) s.Equal(codes.NotFound, st.Code()) s.Equal(stream.ReasonExpired, st.Details()[0].(*streamservice.StreamErrorDetails).GetReason()) // A direct page answers the reason in its operation failure. - _, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: ref(id), FromOffset: first}) + _, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: ref(id), FromOffset: first}, + ) s.Equal(stream.ReasonExpired, causeType(failure)) } @@ -1244,7 +1443,10 @@ func (s *StreamTestSuite) TestDirectReadErrorsKeepTheirReason() { env := s.newEnv() sc := s.streamClient(env) id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: env.Namespace().String(), StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: env.Namespace().String(), StreamId: id}, + ) s.NoError(err) for reason, req := range map[string]*streamservice.ReadRecordsRequest{ @@ -1265,14 +1467,22 @@ func (s *StreamTestSuite) createWithRetention( c *streamCaller, ) (*streamservice.StreamRef, *streamservice.AppendRecordsResponse) { id := testcore.RandomizeStr("chat") - owned := &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID()} + owned := &streamservice.StreamRef{ + Id: id, + WorkflowId: c.run.GetID(), + FirstExecutionRunId: c.run.GetRunID(), + } _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ - Namespace: env.Namespace().String(), StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), + Namespace: env.Namespace(). + String(), + StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), Owner: workflowWriter(c.run.GetRunID(), 3), }) s.NoError(err) appended, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: env.Namespace().String(), Stream: owned, Producer: clientProducer(), Sequence: 1, Records: records("a"), + Namespace: env.Namespace(). + String(), + Stream: owned, Producer: clientProducer(), Sequence: 1, Records: records("a"), }) s.NoError(err) return owned, appended @@ -1287,13 +1497,19 @@ func (s *StreamTestSuite) TestWorkflowWakeKeepsItsFloor() { reader := s.newCaller(env) var woke streamservice.ReadRecordsResponse - failure, _ := reader.outcome(reader.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ - Stream: owned, WakeOnly: true, - }})[0], &woke) + failure, _ := reader.outcome( + reader.schedule(streamOp{"ReadRecords", &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, + }})[0], + &woke, + ) s.Nil(failure) time.Sleep(3 * time.Second) //nolint:forbidigo // The Retention runs on real time. - page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: woke.GetNextOffset()}) + page, failure := s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: woke.GetNextOffset()}, + ) s.Nil(failure) s.Equal([]string{"a"}, kindsAndTopics(page)) } @@ -1304,15 +1520,20 @@ func (s *StreamTestSuite) TestReadLinkToNoRunKeepsNoFloor() { sc := s.streamClient(env) owned, appended := s.createWithRetention(sc, env, s.newCaller(env)) - forged := &commonpb.Link{Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: env.Namespace().String(), WorkflowId: "nobody", RunId: uuid.NewString(), - }}} + forged := &commonpb.Link{ + Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ + Namespace: env.Namespace().String(), WorkflowId: "nobody", RunId: uuid.NewString(), + }}, + } page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned}, forged) s.Nil(failure) s.Equal([]string{"a"}, kindsAndTopics(page)) time.Sleep(3 * time.Second) //nolint:forbidigo // The Retention runs on real time. - _, failure = s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: appended.GetFirstOffset()}) + _, failure = s.directRead( + env, + &streamservice.ReadRecordsRequest{Stream: owned, FromOffset: appended.GetFirstOffset()}, + ) s.Equal(stream.ReasonExpired, causeType(failure)) } @@ -1323,16 +1544,21 @@ func (s *StreamTestSuite) TestDirectReadPastTheReaderCapIsResourceExhausted() { owned, _ := s.createWithRetention(sc, env, s.newCaller(env)) reader := s.newCaller(env).run - _, err := env.FrontendClient().StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ - Namespace: env.Namespace().String(), - Endpoint: commonnexus.SystemEndpoint, - Service: stream.ServiceName, - Operation: "ReadRecords", - Input: payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{Stream: owned}), - Links: []*commonpb.Link{{Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: env.Namespace().String(), WorkflowId: reader.GetID(), RunId: reader.GetRunID(), - }}}}, - }) + _, err := env.FrontendClient(). + StartNexusOperation(s.Context(), &workflowservice.StartNexusOperationRequest{ + Namespace: env.Namespace().String(), + Endpoint: commonnexus.SystemEndpoint, + Service: stream.ServiceName, + Operation: "ReadRecords", + Input: payloads.MustEncodeSingle(&streamservice.ReadRecordsRequest{Stream: owned}), + Links: []*commonpb.Link{ + {Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{ + Namespace: env.Namespace(). + String(), + WorkflowId: reader.GetID(), RunId: reader.GetRunID(), + }}}, + }, + }) var exhausted *serviceerror.ResourceExhausted s.ErrorAs(err, &exhausted) s.Contains(exhausted.Error(), "most Workflow readers") @@ -1379,10 +1605,14 @@ func (s *StreamTestSuite) TestWorkflowReaderOfAClientStreamKeepsItsRangePastTheC sc := s.streamClient(env) ns := env.Namespace().String() id := testcore.RandomizeStr("chat") - _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}) + _, err := sc.CreateStream( + s.Context(), + &streamservice.CreateStreamRequest{Namespace: ns, StreamId: id}, + ) s.NoError(err) _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, Records: records("a"), + Namespace: ns, Stream: ref(id), Producer: clientProducer(), Sequence: 1, + Records: records("a"), }) s.NoError(err) From cbaaf54c808ecf9fe403d09119c23517bf2648fc Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:35:18 -0700 Subject: [PATCH 18/23] Moved two read checks into helpers and shortened round 4 comments. The helpers keep scan and ReadOrPark at or under their complexity before this round. --- chasm/lib/stream/nexus.go | 5 ++-- chasm/lib/stream/read.go | 24 ++++++++++++++----- chasm/lib/stream/readers.go | 5 ++-- chasm/lib/stream/stream.go | 8 +++---- chasm/lib/stream/vectors_test.go | 4 ++-- .../history/timer_queue_task_executor_base.go | 5 ++-- tests/stream_test.go | 11 ++++----- 7 files changed, 33 insertions(+), 29 deletions(-) diff --git a/chasm/lib/stream/nexus.go b/chasm/lib/stream/nexus.go index 3a623f40c94..938cc80feb0 100644 --- a/chasm/lib/stream/nexus.go +++ b/chasm/lib/stream/nexus.go @@ -91,9 +91,8 @@ func newNexusService(h *handler) *nexus.Service { return service } -// readRecordsOperation answers a read at once. A wake_only read with no Record for it is an -// async operation that the Stream completes when Records come or the Stream closes, so a -// Workflow waits on the server with no timer. +// readRecordsOperation answers a read at once, but a wake_only read with no Record is async. The +// Stream completes it on a Record or the close, so a Workflow waits on the server with no timer. type readRecordsOperation struct { nexus.UnimplementedOperation[ *streamservice.ReadRecordsRequest, diff --git a/chasm/lib/stream/read.go b/chasm/lib/stream/read.go index 008170561fe..60a4c976628 100644 --- a/chasm/lib/stream/read.go +++ b/chasm/lib/stream/read.go @@ -106,8 +106,7 @@ func (s *Stream) scan( if end := req.GetToOffset(); end > 0 && entry.GetOffset() >= end { return resp, nil } - full := len(resp.Records) >= limit || (scanned >= maxScan && len(resp.Records) > 0) - if resp.Done || full { + if pageFull(resp, limit, scanned) { return resp, nil } if entry.GetOffset() >= retained && overBytes(req, resp, entry, size) { @@ -131,6 +130,15 @@ func (s *Stream) scan( return resp, nil } +// pageFull says whether a page ends before its next entry. Only a page with Records stops at the +// scan cap, so a reader never takes an empty page for the end with a match left. +func pageFull(resp *streamservice.ReadRecordsResponse, limit int, scanned int) bool { + if len(resp.GetRecords()) == 0 { + return false + } + return resp.GetDone() || len(resp.GetRecords()) >= limit || scanned >= maxScan +} + // firstBatchAt is the index of the first batch with an entry at or past offset, so a look of a // parked read reads only the batches that came since its last look. func (s *Stream) firstBatchAt(offset int64) int { @@ -329,6 +337,11 @@ type ReadInput struct { Limits Limits } +// parks says whether the read may park, a wake_only read whose operation a callback completes. +func (in ReadInput) parks() bool { + return in.Callback != nil && in.Request.GetWakeOnly() +} + // StartForRead is the start of an execution for a read. The Stream waits for its create. The // start parks the read itself, since a child node that the update of a new execution adds is // not persisted. @@ -340,9 +353,8 @@ func StartForRead(ctx chasm.MutableContext, in ReadInput) (*Stream, error) { return s, nil } -// ReadOrPark answers a read at once, but parks a wake_only read that finds no Record until the -// Stream can answer it. It answers nil for a parked read. A wake's answer has no Records, so a -// parked read keeps no page, though one append wakes every parked read in one transaction. +// ReadOrPark answers a read at once, but parks a wake_only read with no Record and answers nil. +// A wake's answer has no Records, so one append that wakes every parked read copies no page. func (s *Stream) ReadOrPark( ctx chasm.MutableContext, in ReadInput, @@ -369,7 +381,7 @@ func (s *Stream) ReadOrPark( if in.Reader != nil { read.ReaderRunId = in.Reader.RunID } - parks := in.Callback != nil && in.Request.GetWakeOnly() + parks := in.parks() wait, createWait := parkWaits(in.Request, in.Limits) if wait > 0 { read.WaitDeadline = timestamppb.New(now.Add(wait)) diff --git a/chasm/lib/stream/readers.go b/chasm/lib/stream/readers.go index f1645fed287..ac6d80c5f24 100644 --- a/chasm/lib/stream/readers.go +++ b/chasm/lib/stream/readers.go @@ -36,9 +36,8 @@ func readerRunOf(links []nexus.Link) *ReaderRun { return nil } -// readerFloor is the floor that a read from from sets for the run. It is false when the run's -// floor is already that low, or when the read fails anyway. A client's Stream keeps floors too, -// since the size cap would otherwise drop what a Workflow's replay reads again. +// readerFloor is the floor that a read from from sets for the run, false when the run's floor is +// as low or the read fails anyway. Every Stream keeps floors, since its size cap trims too. func (s *Stream) readerFloor(ctx chasm.Context, runID string, from int64) (int64, bool) { if from == 0 { from = s.TrimmedOffset + 1 diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index 83294e5822c..419abba6bf3 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -27,8 +27,7 @@ const ( MaxRecords = 1000 // MaxStreamIDBytes is the longest Stream id. MaxStreamIDBytes = 4096 - // The most entries a page with Records scans, since a Topic filter can skip many. An empty - // page scans on to the end, so a reader never takes it for the end with a match left. + // The most entries a page with Records scans, since a Topic filter can skip many. maxScan = 10 * MaxRecords // An offset is the append time in ms shifted by these bits, plus a sequence, as the Redis // Store packs its entry ids. So the offsets of the two backends sort the same way, and the @@ -653,9 +652,8 @@ func (s *Stream) floor() (int64, bool) { return s.Floor, s.Floor > 0 && len(s.Readers) > 0 } -// retainedFrom is the offset below which the Retention hides the Records at now. A nested Stream, -// or one with Workflow readers, hides none that it still stores, so a read from the start begins -// at the same offset on replay. +// retainedFrom is the offset below which the Retention hides the Records at now. A nested Stream +// or one with floors hides none it stores, so a replay of a read from the start starts the same. func (s *Stream) retainedFrom(now time.Time) int64 { retained := s.TrimmedOffset + 1 retention := s.GetRetention().AsDuration() diff --git a/chasm/lib/stream/vectors_test.go b/chasm/lib/stream/vectors_test.go index 610fe747b0c..b2ba1ecb619 100644 --- a/chasm/lib/stream/vectors_test.go +++ b/chasm/lib/stream/vectors_test.go @@ -58,8 +58,8 @@ func readVectors(t *testing.T, name string, into any) { require.NoError(t, json.Unmarshal(data, into)) } -// vectorWithdraws applies a Superseded Record to a Record with the server's rules: the task rule -// of withdrawTasks for a Workflow item, and CompareEpochs for the other kinds that withdraw. +// vectorWithdraws applies a Superseded Record to a Record with the server's rules. A Workflow item +// uses the task rule of withdrawTasks, and the other kinds that withdraw use CompareEpochs. func vectorWithdraws(s *streamservice.Supersession, p *streamservice.Producer) bool { if s.GetWithdrawRun() != "" && p.GetRunId() == s.GetWithdrawRun() { return true diff --git a/service/history/timer_queue_task_executor_base.go b/service/history/timer_queue_task_executor_base.go index da8b42c7017..9655b90c61c 100644 --- a/service/history/timer_queue_task_executor_base.go +++ b/service/history/timer_queue_task_executor_base.go @@ -142,9 +142,8 @@ func (t *timerQueueTaskExecutorBase) executeDeleteHistoryEventTask( if err != nil { return err } - // The first run of a chain keeps the chain's Streams, which the chain may still write after - // this run closed. A Standalone Activity keeps its Streams readable for their Retention. The - // execution stays one more retention, until its Streams expire. + // A chain writes its first run's Streams after that run closed, and a closed Standalone + // Activity keeps its Streams for their Retention. So the execution waits one more retention. if holds { mutableState.AddTasks(&tasks.DeleteHistoryEventTask{ WorkflowKey: task.WorkflowKey, diff --git a/tests/stream_test.go b/tests/stream_test.go index 9ed123f48c0..90d7553a404 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -587,9 +587,8 @@ func (s *StreamTestSuite) TestWorkflowWakeWaitsForTheCreate() { s.Empty(page.GetRecords()) } -// A wake whose wait for the create ends first leaves the Workflow Stream's node abandoned. The -// node has no execution of its own to replace, so a later wake parks again and the create opens -// it. +// A wake whose create wait ends first leaves the Workflow Stream's node abandoned. A later wake +// parks again and the create opens it, since the node has no execution of its own to replace. func (s *StreamTestSuite) TestAbandonedWorkflowStreamCanStillBeCreated() { env := s.newEnv() sc := s.streamClient(env) @@ -1694,10 +1693,8 @@ func localActivityWriter(runID string, started int64) *streamservice.Producer { } } -// A caller with write access can't close a Workflow Stream, write as a run outside its chain, -// fence a run it doesn't follow, or put a Stream into another Workflow's first run. A Local -// Activity of the chain can't close it either, since it writes before its task commits. A -// Workflow owns no Stream outside its chain. +// A caller with write access can't close a Workflow Stream, write or fence as another run, or +// create a Workflow Stream outside its chain. A Local Activity of the chain can't close it. func (s *StreamTestSuite) TestForgedOwnerCallsAreRefused() { env := s.newEnv() sc := s.streamClient(env) From e5c628b892dc9f55fd1804a4c4a64eef97cbfa90 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:52:05 -0700 Subject: [PATCH 19/23] Ran a nested Stream's tasks under a closed host. CHASM dropped the tasks of a Stream in a first run after continue-as-new or in a closed Activity. Its expiry, wake waits and readers check never ran, so the hold kept the host forever. --- chasm/lib/activity/stream.go | 12 +- chasm/lib/stream/host.go | 20 ++-- chasm/lib/workflow/workflow.go | 8 +- tests/stream_test.go | 206 +++++++++++++++++++++++++-------- 4 files changed, 171 insertions(+), 75 deletions(-) diff --git a/chasm/lib/activity/stream.go b/chasm/lib/activity/stream.go index d38f166fbb3..c91d8de227a 100644 --- a/chasm/lib/activity/stream.go +++ b/chasm/lib/activity/stream.go @@ -82,17 +82,9 @@ func (a *Activity) checkAttempt( } // HoldsStreams says whether a Stream of the closed Activity is still readable, so the Activity -// outlives the namespace retention that would delete it. Each Stream closed with the Activity. +// outlives the namespace retention. A closed Stream stays readable for its Retention. func (a *Activity) HoldsStreams(ctx chasm.Context) bool { - now := ctx.Now(a) - for _, field := range a.Streams { - if s, ok := field.TryGet(ctx); ok { - if until, held := s.HeldUntil(); held && until.After(now) { - return true - } - } - } - return false + return stream.HoldsStreams(ctx, a.Streams) } // endStreams closes the Activity's Streams with its end. Each terminal transition passes its diff --git a/chasm/lib/stream/host.go b/chasm/lib/stream/host.go index 37eb7668272..3f07f3e9ef3 100644 --- a/chasm/lib/stream/host.go +++ b/chasm/lib/stream/host.go @@ -3,7 +3,6 @@ package stream import ( "context" "errors" - "time" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -65,13 +64,16 @@ func (s *Stream) EndChain(ctx chasm.MutableContext, end RunEnd, limits Limits) { s.finish(ctx, limits) } -// HeldUntil is when a closed Stream expires, false for a Stream that isn't closed. A closed host -// runs no task of its Streams, so the close time and the Retention tell when the Stream expires. -func (s *Stream) HeldUntil() (time.Time, bool) { - if s.Phase != streampb.STREAM_PHASE_CLOSED { - return time.Time{}, false +// HoldsStreams says whether a Stream of a closed host is still readable, so the host must outlive +// the namespace retention that would delete it. +func HoldsStreams(ctx chasm.Context, streams chasm.Map[string, *Stream]) bool { + for _, field := range streams { + if s, ok := field.TryGet(ctx); ok && s.Phase != streampb.STREAM_PHASE_EXPIRED && + s.Phase != streampb.STREAM_PHASE_ABANDONED { + return true + } } - return s.closedExpiryAt(), true + return false } // EndOwner closes a Standalone Activity's Stream with the Activity's end, in the transition that @@ -196,7 +198,9 @@ func update[O any]( if err != nil { return zero, err } - streams[t.id] = chasm.NewComponentField(mctx, s) + // A Stream outlives its host's close, as in a chain's first run or an ended Activity, + // so its timers, wakes and readers check must run under a closed host. + streams[t.id] = chasm.NewComponentField(mctx, s, chasm.ComponentFieldDetached()) return out, nil }, struct{}{}) return out, uncarry(err) diff --git a/chasm/lib/workflow/workflow.go b/chasm/lib/workflow/workflow.go index 3f52aa39f18..f7a5c3e41cd 100644 --- a/chasm/lib/workflow/workflow.go +++ b/chasm/lib/workflow/workflow.go @@ -133,13 +133,7 @@ func (w *Workflow) HasChainStreams(ctx chasm.Context, firstRunID string) bool { // namespace retention that would delete it. A chain that continues as new keeps writing to the // Streams of its first run. func (w *Workflow) HoldsStreams(ctx chasm.Context) bool { - for _, field := range w.Streams { - if s, ok := field.TryGet(ctx); ok && s.Phase != streampb.STREAM_PHASE_EXPIRED && - s.Phase != streampb.STREAM_PHASE_ABANDONED { - return true - } - } - return false + return stream.HoldsStreams(ctx, w.Streams) } // EndChainStreams closes the chain's close-with-run Streams with this run's close event. The diff --git a/tests/stream_test.go b/tests/stream_test.go index 90d7553a404..ed1c3f74d81 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -1,6 +1,7 @@ package tests import ( + "errors" "testing" "time" @@ -1157,24 +1158,7 @@ func (s *StreamTestSuite) TestWorkflowReadsDirectlyWithNoRecordsInHistory() { // The first run of a chain keeps the chain's Streams after it closes, so the namespace retention // can't delete them while a later run still writes. func (s *StreamTestSuite) TestChainOutlivesTheRetentionOfItsFirstRun() { - env := testcore.NewEnv(s.T(), - testcore.WithDedicatedCluster(), - testcore.WithDynamicConfig(stream.Enabled, true), - testcore.WithDynamicConfig(dynamicconfig.NamespaceMinRetentionGlobal, time.Duration(0)), - testcore.WithDynamicConfig(dynamicconfig.NamespaceMinRetentionLocal, time.Duration(0)), - testcore.WithDynamicConfig(dynamicconfig.RetentionTimerJitterDuration, time.Duration(0)), - ) - _, err := env.FrontendClient(). - UpdateNamespace(s.Context(), &workflowservice.UpdateNamespaceRequest{ - Namespace: env.Namespace().String(), - Config: &namespacepb.NamespaceConfig{ - WorkflowExecutionRetentionTtl: durationpb.New(time.Second), - }, - }) - s.NoError(err) - // Wait for the namespace cache to see the new retention. - <-time.After(2 * testcore.NamespaceCacheRefreshInterval) - + env := s.retentionEnv() sc := s.streamClient(env) c := s.newCaller(env) owned := s.closeWithRun(sc, env, c) @@ -1182,7 +1166,7 @@ func (s *StreamTestSuite) TestChainOutlivesTheRetentionOfItsFirstRun() { // Several retentions of the first run pass while run 2 is open. <-time.After(4 * time.Second) - _, err = sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ + _, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ Namespace: env.Namespace().String(), Stream: owned, Producer: workflowWriter(c.current, 3), Sequence: 1, Records: records("late"), }) @@ -1195,9 +1179,9 @@ func (s *StreamTestSuite) TestChainOutlivesTheRetentionOfItsFirstRun() { s.Equal([]string{"late"}, kindsAndTopics(page)) } -// A closed Standalone Activity keeps its Streams readable for their Retention, so the namespace -// retention can't delete them with the Activity. -func (s *StreamTestSuite) TestActivityStreamOutlivesTheRetentionOfItsActivity() { +// retentionEnv is a cluster whose namespace retention is 1 s, so a retention delete shows in +// seconds. +func (s *StreamTestSuite) retentionEnv() *testcore.TestEnv { env := testcore.NewEnv(s.T(), testcore.WithDedicatedCluster(), testcore.WithDynamicConfig(stream.Enabled, true), @@ -1205,18 +1189,31 @@ func (s *StreamTestSuite) TestActivityStreamOutlivesTheRetentionOfItsActivity() testcore.WithDynamicConfig(dynamicconfig.NamespaceMinRetentionLocal, time.Duration(0)), testcore.WithDynamicConfig(dynamicconfig.RetentionTimerJitterDuration, time.Duration(0)), ) - ctx := s.Context() - ns := env.Namespace().String() - _, err := env.FrontendClient().UpdateNamespace(ctx, &workflowservice.UpdateNamespaceRequest{ - Namespace: ns, - Config: &namespacepb.NamespaceConfig{ - WorkflowExecutionRetentionTtl: durationpb.New(time.Second), - }, - }) + _, err := env.FrontendClient().UpdateNamespace(s.Context(), + &workflowservice.UpdateNamespaceRequest{ + Namespace: env.Namespace().String(), + Config: &namespacepb.NamespaceConfig{ + WorkflowExecutionRetentionTtl: durationpb.New(time.Second), + }, + }) s.NoError(err) // Wait for the namespace cache to see the new retention. <-time.After(2 * testcore.NamespaceCacheRefreshInterval) + return env +} + +// activityStream is a Stream of a Standalone Activity's first attempt, with one Record. +type activityStream struct { + ref *streamservice.StreamRef + task *workflowservice.PollActivityTaskQueueResponse +} +func (s *StreamTestSuite) startActivityStream( + env *testcore.TestEnv, + retention *durationpb.Duration, +) activityStream { + ctx := s.Context() + ns := env.Namespace().String() activityID := "tool-" + uuid.NewString() taskQueue := &taskqueuepb.TaskQueue{ Name: "tools-" + uuid.NewString(), @@ -1234,27 +1231,25 @@ func (s *StreamTestSuite) TestActivityStreamOutlivesTheRetentionOfItsActivity() s.NoError(err) task, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ - Namespace: ns, - TaskQueue: taskQueue, - Identity: "test", + Namespace: ns, TaskQueue: taskQueue, Identity: "test", }) s.NoError(err) id := testcore.RandomizeStr("tokens") owned := &streamservice.StreamRef{ - Id: id, - ActivityId: activityID, - ActivityRunId: started.GetRunId(), + Id: id, ActivityId: activityID, ActivityRunId: started.GetRunId(), } owner := &streamservice.Producer{ Kind: streamservice.PRODUCER_KIND_STANDALONE_ACTIVITY, Id: "sa:" + activityID + ":" + started.GetRunId(), Epoch: &streamservice.ProducerEpoch{ - AttemptScheduledTime: task.GetCurrentAttemptScheduledTime(), Attempt: task.GetAttempt(), + AttemptScheduledTime: task.GetCurrentAttemptScheduledTime(), + Attempt: task.GetAttempt(), }, } sc := s.streamClient(env) _, err = sc.CreateStream(ctx, &streamservice.CreateStreamRequest{ - Namespace: ns, StreamId: id, Ref: owned, Owner: owner, TaskToken: task.GetTaskToken(), + Namespace: ns, StreamId: id, Ref: owned, Owner: owner, Retention: retention, + TaskToken: task.GetTaskToken(), }) s.NoError(err) _, err = sc.AppendRecords(ctx, &streamservice.AppendRecordsRequest{ @@ -1262,22 +1257,135 @@ func (s *StreamTestSuite) TestActivityStreamOutlivesTheRetentionOfItsActivity() TaskToken: task.GetTaskToken(), }) s.NoError(err) - _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, + return activityStream{ref: owned, task: task} +} + +// completeActivity completes the attempt of an Activity Stream, which closes the Stream. +func (s *StreamTestSuite) completeActivity(env *testcore.TestEnv, a activityStream) { + _, err := env.FrontendClient().RespondActivityTaskCompleted(s.Context(), &workflowservice.RespondActivityTaskCompletedRequest{ - Namespace: ns, TaskToken: task.GetTaskToken(), Identity: "test", + Namespace: env.Namespace().String(), TaskToken: a.task.GetTaskToken(), Identity: "test", }) s.NoError(err) +} + +// A closed Standalone Activity keeps its Streams readable for their Retention, so the namespace +// retention can't delete them with the Activity. +func (s *StreamTestSuite) TestActivityStreamOutlivesTheRetentionOfItsActivity() { + env := s.retentionEnv() + a := s.startActivityStream(env, nil) + s.completeActivity(env, a) // Several namespace retentions pass after the Activity's end. <-time.After(4 * time.Second) - page, err := sc.ReadRecords( - ctx, - &streamservice.ReadRecordsRequest{Namespace: ns, Stream: owned}, - ) + page, err := s.streamClient(env).ReadRecords(s.Context(), + &streamservice.ReadRecordsRequest{Namespace: env.Namespace().String(), Stream: a.ref}) s.NoError(err) s.Equal([]string{"tokens", "close"}, kindsAndTopics(page)) } +// A closed Activity's Stream expires on its own timer, and then the namespace retention deletes +// the Activity, though nobody reads the Stream. +func (s *StreamTestSuite) TestClosedActivityIsDeletedAfterItsStreamExpires() { + env := s.retentionEnv() + a := s.startActivityStream(env, durationpb.New(time.Second)) + s.completeActivity(env, a) + + s.AwaitTrue(func() bool { + _, err := env.FrontendClient().DescribeActivityExecution(s.Context(), + &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: a.ref.GetActivityId(), + RunId: a.ref.GetActivityRunId(), + }) + var notFound *serviceerror.NotFound + return errors.As(err, ¬Found) + }, 20*time.Second, 200*time.Millisecond) +} + +// A chain's first run keeps its Streams after it continues as new. Their tasks still run there, +// so an unread closed Stream expires, and then the namespace retention deletes the first run. +func (s *StreamTestSuite) TestUnreadFirstRunIsDeletedAfterItsStreamExpires() { + env := s.retentionEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + ns := env.Namespace().String() + id := testcore.RandomizeStr("chat") + first := c.run.GetRunID() + owned := &streamservice.StreamRef{Id: id, WorkflowId: c.run.GetID(), FirstExecutionRunId: first} + _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ + Namespace: ns, StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), + Owner: workflowWriter(first, 3), CloseWithRun: true, + }) + s.NoError(err) + c.respond(c.continueAsNew()) + s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), c.run.GetID(), c.current, "stop")) + + s.AwaitTrue(func() bool { + _, err := env.AdminClient().DescribeMutableState(s.Context(), + &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: c.run.GetID(), RunId: first}, + Archetype: chasm.WorkflowArchetype, + }) + var notFound *serviceerror.NotFound + return errors.As(err, ¬Found) + }, 20*time.Second, 200*time.Millisecond) +} + +// A wake on a Stream of a first run that continued as new ends at its wait, since the Stream's +// tasks run under the closed run. +func (s *StreamTestSuite) TestWakeInAClosedFirstRunEndsAtItsWait() { + env := s.newEnv() + sc := s.streamClient(env) + c := s.newCaller(env) + owned := s.closeWithRun(sc, env, c) + c.respond(c.continueAsNew()) + + reader := s.newCaller(env) + var woke streamservice.ReadRecordsResponse + failure, started := reader.outcome(reader.schedule(streamOp{"ReadRecords", + &streamservice.ReadRecordsRequest{ + Stream: owned, WakeOnly: true, Wait: durationpb.New(2 * time.Second), + }})[0], &woke) + s.Nil(failure) + s.True(started) + s.Empty(woke.GetRecords()) + s.False(woke.GetDone()) +} + +// The readers check runs in a first run that continued as new, so the floor of a reader that +// closed goes, and the Retention trims its range. +func (s *StreamTestSuite) TestReadersCheckRunsInAClosedFirstRun() { + env := s.newEnv(testcore.WithDynamicConfig(stream.ReaderCheckInterval, time.Second)) + sc := s.streamClient(env) + c := s.newCaller(env) + readerRun := s.newCaller(env).run + owned, appended := s.createWithRetention(sc, env, c) + page, failure := s.directRead(env, &streamservice.ReadRecordsRequest{Stream: owned}, + readerLink(env, readerRun)) + s.Nil(failure) + s.Equal([]string{"a"}, kindsAndTopics(page)) + c.respond(c.continueAsNew()) + + // Past the Retention, the floor still keeps the range. + time.Sleep(2 * time.Second) //nolint:forbidigo // The Retention runs on real time. + again, failure := s.directRead(env, &streamservice.ReadRecordsRequest{ + Stream: owned, ToOffset: page.GetNextOffset(), + }) + s.Nil(failure) + s.Equal(page.GetRecords(), again.GetRecords()) + s.NoError(env.SdkClient().TerminateWorkflow(s.Context(), readerRun.GetID(), + readerRun.GetRunID(), "reader done")) + + s.AwaitTrue(func() bool { + _, failure := s.directRead(env, &streamservice.ReadRecordsRequest{ + Stream: owned, FromOffset: appended.GetFirstOffset(), + }) + return failure != nil && causeType(failure) == stream.ReasonExpired + }, 10*time.Second, 200*time.Millisecond) +} + // An append that names a Workflow Stream by its bare id, or a Stream that doesn't exist, fails. // It never writes somewhere a reader of the Workflow Stream can't see. func (s *StreamTestSuite) TestAppendToAStreamThatIsNotThereFails() { @@ -1471,17 +1579,15 @@ func (s *StreamTestSuite) createWithRetention( WorkflowId: c.run.GetID(), FirstExecutionRunId: c.run.GetRunID(), } + ns := env.Namespace().String() _, err := sc.CreateStream(s.Context(), &streamservice.CreateStreamRequest{ - Namespace: env.Namespace(). - String(), - StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), + Namespace: ns, StreamId: id, Ref: owned, Retention: durationpb.New(time.Second), Owner: workflowWriter(c.run.GetRunID(), 3), }) s.NoError(err) appended, err := sc.AppendRecords(s.Context(), &streamservice.AppendRecordsRequest{ - Namespace: env.Namespace(). - String(), - Stream: owned, Producer: clientProducer(), Sequence: 1, Records: records("a"), + Namespace: ns, Stream: owned, Producer: clientProducer(), Sequence: 1, + Records: records("a"), }) s.NoError(err) return owned, appended From 4c2a8f11c763c138a219b3fd80a4e297ea35e4ae Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 02:46:41 -0700 Subject: [PATCH 20/23] Tightened the comments and docs on this branch. --- chasm/lib/activity/activity_test.go | 4 +-- chasm/lib/activity/stream.go | 21 ++++++------ chasm/lib/stream/frontend.go | 6 ++-- .../lib/stream/gen/streampb/v1/message.pb.go | 2 +- chasm/lib/stream/handler.go | 27 +++++++-------- chasm/lib/stream/host.go | 29 ++++++++-------- chasm/lib/stream/nexus.go | 8 ++--- chasm/lib/stream/owner.go | 22 +++++-------- chasm/lib/stream/producer.go | 33 ++++++++----------- chasm/lib/stream/proto/v1/message.proto | 2 +- chasm/lib/stream/proto/v1/service.proto | 2 +- chasm/lib/stream/read.go | 29 ++++++++-------- chasm/lib/stream/readers.go | 15 ++++----- chasm/lib/stream/state.go | 8 ++--- chasm/lib/stream/stream.go | 22 ++++++------- chasm/lib/stream/stream_test.go | 18 +++++----- chasm/lib/workflow/workflow.go | 8 ++--- common/rpc/interceptor/namespace_validator.go | 4 +-- service/frontend/nexus_direct.go | 9 +++-- tests/stream_test.go | 10 +++--- 20 files changed, 131 insertions(+), 148 deletions(-) diff --git a/chasm/lib/activity/activity_test.go b/chasm/lib/activity/activity_test.go index 5d3674c9dd7..fe75eef8c6a 100644 --- a/chasm/lib/activity/activity_test.go +++ b/chasm/lib/activity/activity_test.go @@ -1149,8 +1149,8 @@ func TestEffectiveUserMetadata_FallsBackToLegacy(t *testing.T) { require.Same(t, legacyMD, got) } -// A Stream ref with an activity id routes to the Activity's component, so the stream library's -// copy of the archetype must stay this library's. +// A Stream ref with an activity id routes to stream.ActivityArchetypeID, so that id must equal +// this library's ArchetypeID. func TestStreamHostArchetypeIsTheActivity(t *testing.T) { require.Equal(t, ArchetypeID, stream.ActivityArchetypeID) } diff --git a/chasm/lib/activity/stream.go b/chasm/lib/activity/stream.go index c91d8de227a..d5b5f4cc9db 100644 --- a/chasm/lib/activity/stream.go +++ b/chasm/lib/activity/stream.go @@ -8,9 +8,8 @@ import ( "go.temporal.io/server/common/tasktoken" ) -// A Standalone Activity owns its Streams. They nest in the Activity's execution, so a call routes -// by the activity id, and they close in the transition that ends the Activity, with its status. -// A retry is the same execution, so a Stream goes on across attempts. +// A Standalone Activity's Streams nest in its execution, so a call routes by the activity id. A +// retry is the same execution, so a Stream stays open across attempts. var _ stream.Host = (*Activity)(nil) @@ -22,8 +21,8 @@ func (a *Activity) NestedStreams() chasm.Map[string, *stream.Stream] { return a.Streams } -// CheckOwner refuses a create that isn't the Activity's own, by its current attempt. It -// implements [stream.Host]. +// CheckOwner refuses a create unless the owner is the Activity's own Producer with the task token +// of its current attempt. It implements [stream.Host]. func (a *Activity) CheckOwner( ctx chasm.Context, owner *streamservice.Producer, @@ -35,8 +34,8 @@ func (a *Activity) CheckOwner( return a.checkAttempt(ctx, owner, token) } -// CheckWriter refuses a write as the Activity's own Producer by any other than its current -// attempt. Other Producers write as on any Stream. It implements [stream.Host]. +// CheckWriter refuses a write as the Activity's own Producer unless the task token is the +// Activity's current attempt. It doesn't check other Producers. It implements [stream.Host]. func (a *Activity) CheckWriter( ctx chasm.Context, producer *streamservice.Producer, @@ -48,7 +47,7 @@ func (a *Activity) CheckWriter( return a.checkAttempt(ctx, producer, token) } -// ownProducer says whether a Producer is this Activity's, by the id that the crate makes. +// ownProducer says whether a Producer has the kind and the id that the crate gives this Activity. func (a *Activity) ownProducer(ctx chasm.Context, producer *streamservice.Producer) bool { key := ctx.ExecutionKey() id := "sa:" + stream.EncodeIDPart(key.BusinessID) + ":" + stream.EncodeIDPart(key.RunID) @@ -57,7 +56,7 @@ func (a *Activity) ownProducer(ctx chasm.Context, producer *streamservice.Produc } // checkAttempt checks that a task token is this Activity's current attempt, and that the -// Producer's epoch is that attempt. The epoch stays (attempt scheduled time, attempt). +// Producer's epoch has the same attempt. func (a *Activity) checkAttempt( ctx chasm.Context, producer *streamservice.Producer, @@ -82,12 +81,12 @@ func (a *Activity) checkAttempt( } // HoldsStreams says whether a Stream of the closed Activity is still readable, so the Activity -// outlives the namespace retention. A closed Stream stays readable for its Retention. +// must outlive the namespace retention. A closed Stream stays readable for its Retention. func (a *Activity) HoldsStreams(ctx chasm.Context) bool { return stream.HoldsStreams(ctx, a.Streams) } -// endStreams closes the Activity's Streams with its end. Each terminal transition passes its +// endStreams closes the Activity's Streams when it ends. Each terminal transition passes its // status, since the transition sets the Activity's status after its apply function. func (a *Activity) endStreams(ctx chasm.MutableContext, status activitypb.ActivityExecutionStatus) { if len(a.Streams) == 0 { diff --git a/chasm/lib/stream/frontend.go b/chasm/lib/stream/frontend.go index 958e0595066..95978ea2c12 100644 --- a/chasm/lib/stream/frontend.go +++ b/chasm/lib/stream/frontend.go @@ -28,9 +28,9 @@ func NewFrontendHandler( return &FrontendHandler{client: client, config: config, namespaceRegistry: namespaceRegistry} } -// RedirectableMethods lists the StreamService methods that a cell forwards to the namespace's -// active cell, with the response each one answers. The Stream is in the active cell's -// mutable state, so a call served elsewhere would read a replica or write a copy. +// RedirectableMethods maps each StreamService method that a cell forwards to the namespace's +// active cell to a func that makes its empty response. The Stream is in the active cell's mutable +// state, so a call served elsewhere would read a replica or write a copy. func RedirectableMethods() map[string]func() any { return map[string]func() any{ streamservice.StreamService_CreateStream_FullMethodName: func() any { diff --git a/chasm/lib/stream/gen/streampb/v1/message.pb.go b/chasm/lib/stream/gen/streampb/v1/message.pb.go index 1db28d19bf0..a8930386a8b 100644 --- a/chasm/lib/stream/gen/streampb/v1/message.pb.go +++ b/chasm/lib/stream/gen/streampb/v1/message.pb.go @@ -665,7 +665,7 @@ type ParkedReadState struct { Request *v1.ReadRecordsRequest `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` // The Nexus request id, so a retry of the start finds this read. RequestId string `protobuf:"bytes,2,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` - // Where the next look starts. A look that finds only other Topics moves it on. + // Where the next look starts. A look that finds only other Topics advances it. ScanOffset int64 `protobuf:"varint,3,opt,name=scan_offset,json=scanOffset,proto3" json:"scan_offset,omitempty"` // Set while the read waits for the create. CreateDeadline *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=create_deadline,json=createDeadline,proto3" json:"create_deadline,omitempty"` diff --git a/chasm/lib/stream/handler.go b/chasm/lib/stream/handler.go index d11d38aacbc..a5be948d855 100644 --- a/chasm/lib/stream/handler.go +++ b/chasm/lib/stream/handler.go @@ -38,7 +38,7 @@ func newHandler( // Reader is a Workflow read through System Nexus. type Reader struct { - // The Nexus request id, which a retry of the operation start carries again. + // The Nexus request id. A retry of the operation start sends the same id. RequestID string // The completion callback of the read's operation. A read with none answers at once. Callback *commonpb.Callback @@ -299,10 +299,9 @@ func (h *handler) CloseStream( return resp, streamError(ctx, t, err) } -// CloseFromRun closes a Workflow Stream for the System Nexus command of run, the calling run -// from the link that the server's Nexus call adds. Run must be an open run of the Stream's chain, -// and the close goes in as run's Workflow Producer. Only the result, the failure and the status -// come from the request. +// CloseFromRun closes a Workflow Stream for the System Nexus command of run. Run comes from the +// link that the server's Nexus call adds, and must be an open run of the Stream's chain. The close +// uses run's Workflow Producer, and takes only the result, the failure and the status from req. func (h *handler) CloseFromRun( ctx context.Context, req *streamservice.CloseStreamRequest, @@ -457,12 +456,12 @@ func (h *handler) CancelRead( const pollTick = 100 * time.Millisecond // longPollBuffer is the time a read leaves before the caller's deadline, so the empty answer -// gets back in time. +// reaches the caller before its deadline. const longPollBuffer = time.Second -// ReadRecords is a frontend read. It waits up to create_wait for the create and up to wait for -// a Record, and answers an empty page after the wait. A page of an open Workflow run, run, keeps -// the run's floor first, so no trim drops what it reads. +// ReadRecords is a frontend long poll. It waits up to create_wait for the create and up to wait +// for a Record. The first page of an open Workflow run also sets the run's floor, so no trim +// drops the Records that the run read. func (h *handler) ReadRecords( ctx context.Context, req *streamservice.ReadRecordsRequest, @@ -529,9 +528,8 @@ func (h *handler) ReadRecords( } } -// floorRun is the run whose floor the first page keeps, or nil. That is a run whose page would -// lower its floor, mostly its first, and that is open. The link is the Worker's claim, so a run -// that is closed or doesn't exist keeps no floor. +// floorRun gives run if the first page would lower its floor and the run is open, or nil. The +// Worker sets the link, so a run that is closed or doesn't exist gets no floor. func (h *handler) floorRun( ctx context.Context, t target, @@ -556,9 +554,8 @@ func (h *handler) floorRun( return run, nil } -// page reads one page from from. With floorRun set, the transaction that answers the page keeps -// the run's floor too, so no trim comes between the answer and the floor. The page of a reader -// run suggests continue-as-new when the run's floor keeps too much. +// page reads one page at from. With floorRun set, the transaction that answers the page also keeps +// the run's floor, so no trim runs between the answer and the floor. func (h *handler) page( ctx context.Context, t target, diff --git a/chasm/lib/stream/host.go b/chasm/lib/stream/host.go index 3f07f3e9ef3..1a1f7a450cb 100644 --- a/chasm/lib/stream/host.go +++ b/chasm/lib/stream/host.go @@ -17,16 +17,16 @@ import ( "go.temporal.io/server/common/nexus/nexusrpc" ) -// Host is a component that nests Streams: the Workflow, for the Streams of its run chain, or a -// Standalone Activity, for its own. A nested Stream commits in its host's transaction, so an -// append and the wake of its readers go together. +// Host is a component that nests Streams. A Workflow hosts the Streams of its run chain, and a +// Standalone Activity hosts its own. A nested Stream commits in its host's transaction, so an +// append and the wake of its readers commit together. type Host interface { chasm.Component NestedStreams() chasm.Map[string, *Stream] - // CheckOwner refuses a create whose owner the host doesn't prove, with the task token of its - // current attempt. A Workflow's chain is checked before the transaction. + // CheckOwner refuses a create whose owner isn't the host. A Standalone Activity checks the + // task token of its current attempt. CheckOwner(ctx chasm.Context, owner *streamservice.Producer, token []byte) error - // CheckWriter refuses a write that claims the host's own Producer with no proof. + // CheckWriter refuses a write as the host's own Producer from any other caller. CheckWriter(ctx chasm.Context, producer *streamservice.Producer, token []byte) error } @@ -37,8 +37,8 @@ type ChainHost interface { RunEnd(ctx chasm.Context) (RunEnd, error) } -// ActivityArchetypeID is the archetype of the Activity library's component, which hosts a -// Standalone Activity's Streams. The activity library imports this one, so the id is made here. +// ActivityArchetypeID is the archetype of the Activity library's component. The activity library +// imports this package, so this package can't take the id from there. var ActivityArchetypeID = chasm.GenerateTypeID(chasm.FullyQualifiedName("activity", "activity")) // RunEnd is how the run that ends a chain ended. The chain's close-with-run Streams close with it. @@ -77,7 +77,7 @@ func HoldsStreams(ctx chasm.Context, streams chasm.Map[string, *Stream]) bool { } // EndOwner closes a Standalone Activity's Stream with the Activity's end, in the transition that -// ends it. A retry doesn't end the Activity, so the Stream goes on across attempts. +// ends it. A retry doesn't end the Activity, so the Stream stays open across attempts. func (s *Stream) EndOwner(ctx chasm.MutableContext, end RunEnd, limits Limits) { if s.Phase == streampb.STREAM_PHASE_OPEN { s.close(ctx, s.GetOwner(), end.Result, end.Failure, end.Status) @@ -85,8 +85,8 @@ func (s *Stream) EndOwner(ctx chasm.MutableContext, end RunEnd, limits Limits) { s.finish(ctx, limits) } -// target is where a Stream lives: its own execution keyed by the Stream id, or a child of its -// host, the Workflow component of its chain's first run or a Standalone Activity. +// target is where a Stream lives. That's its own execution keyed by the Stream id, or a child of +// its host, which is the Workflow component of its chain's first run or a Standalone Activity. type target struct { key chasm.ExecutionKey id string @@ -105,7 +105,7 @@ func (t target) activity() bool { } // targetOf finds the Stream that a ref names. A Workflow Stream nests in the first run of its -// chain, since a run's CHASM tree doesn't move to the next run. +// chain, since each run of a chain has its own CHASM tree. func targetOf(namespaceID, streamID string, ref *streamservice.StreamRef) (target, error) { if ref.GetWorkflowId() != "" && ref.GetActivityId() != "" { return target{}, invalid("A Stream ref has a Workflow or an Activity, not both") @@ -144,7 +144,8 @@ func targetOf(namespaceID, streamID string, ref *streamservice.StreamRef) (targe }, nil } -// newNestedStream makes a nested Stream, whose Retention hides no Record that it still stores. +// newNestedStream makes a nested Stream. Its reads see every Record it stores, also past the +// Retention. func newNestedStream(ctx chasm.MutableContext) (*Stream, error) { s := NewStream(ctx, streampb.STREAM_PHASE_PENDING) s.Nested = true @@ -199,7 +200,7 @@ func update[O any]( return zero, err } // A Stream outlives its host's close, as in a chain's first run or an ended Activity, - // so its timers, wakes and readers check must run under a closed host. + // so its timers, wakes and reader checks must run under a closed host. streams[t.id] = chasm.NewComponentField(mctx, s, chasm.ComponentFieldDetached()) return out, nil }, struct{}{}) diff --git a/chasm/lib/stream/nexus.go b/chasm/lib/stream/nexus.go index 938cc80feb0..f6728b67a0d 100644 --- a/chasm/lib/stream/nexus.go +++ b/chasm/lib/stream/nexus.go @@ -32,9 +32,8 @@ func readerOf(options nexus.StartOperationOptions) Reader { return reader } -// nexusError gives the Nexus form of a Stream error. A Stream error fails the operation with -// an application failure whose type is the reason token, so the Workflow sees the same error -// class as a client. Any other error stays a handler error, which the caller retries. +// nexusError converts a Stream error to an application failure whose type is the reason token, so +// a Workflow sees the same error class as a client. Other errors stay handler errors for retry. func nexusError(err error) error { if err == nil { return nil @@ -53,8 +52,7 @@ func nexusError(err error) error { } // newNexusService serves the reads and the close to Workflow code. The close is a command, so the -// server runs it only when the Workflow Task that issued it commits. Workflow writes go over the -// frontend from the Core extension, which knows the Workflow Task facts of each write. +// server runs it only when the Workflow Task that issued it commits. func newNexusService(h *handler) *nexus.Service { service := nexus.NewService(ServiceName) service.MustRegister( diff --git a/chasm/lib/stream/owner.go b/chasm/lib/stream/owner.go index e855cdb7b23..75218b2f463 100644 --- a/chasm/lib/stream/owner.go +++ b/chasm/lib/stream/owner.go @@ -14,8 +14,7 @@ import ( "go.temporal.io/server/common/tasktoken" ) -// The checks of a Stream's owner and of the Producers that claim a Workflow run or an Activity -// attempt. A caller with write access could otherwise close any Stream, write as a Workflow, +// Without these checks, a caller with write access could close any Stream, write as a Workflow, // fence a run, or put a Stream into a Workflow's first run, which then outlives its retention. func notOwner(format string, args ...any) error { @@ -30,9 +29,8 @@ func ownerKind(owner *streamservice.Producer) streamservice.ProducerKind { return streamservice.PRODUCER_KIND_CLIENT } -// checkOwner refuses a create whose owner can't own a Stream. A Workflow Stream nests in its -// chain's first run, so only that chain creates one. A Standalone Activity's Stream nests in the -// Activity, so only the Activity creates one, and its host checks the task token. +// checkOwner refuses a create whose owner can't own a Stream. A nested Stream is a child of its +// host, so only the host creates it. An Activity host checks the task token as well. func checkOwner(owner *streamservice.Producer, t target) error { switch kind := ownerKind(owner); { case t.workflow() && kind != streamservice.PRODUCER_KIND_WORKFLOW: @@ -54,9 +52,8 @@ func checkOwner(owner *streamservice.Producer, t target) error { } } -// checkCloser refuses a close that the Stream's owner doesn't allow. A Workflow Stream closes -// only as a run of its chain, and a client's Stream only from a client. The server's own close -// with the run chain comes from no caller. +// checkCloser refuses a close by a Producer of another kind than the Stream's owner. The close +// with the run chain comes from the server, so it doesn't go through this check. func (s *Stream) checkCloser(producer *streamservice.Producer) error { kind := producer.GetKind() owner := ownerKind(s.GetOwner()) @@ -88,8 +85,8 @@ func ofRun(kind streamservice.ProducerKind) bool { kind == streamservice.PRODUCER_KIND_LOCAL_ACTIVITY } -// checkRunID refuses a Producer whose id doesn't carry its run, so a write can't claim a run in -// one field and show another in its id. +// checkRunID refuses a Producer whose id doesn't contain its run id, so a write can't claim one +// run in run_id and another in its id. func checkRunID(producer *streamservice.Producer) error { run := encodePart(producer.GetRunId()) id := producer.GetId() @@ -129,7 +126,7 @@ func (h *handler) chainRun(ctx context.Context, t target, runID string) (bool, e resp.GetWorkflowStatus() == enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, nil } -// nextRun says whether next is the run that follows run in the chain, so only a retry run +// nextRun says whether next is the run after run in the chain, so only a retry run // withdraws its previous run. func (h *handler) nextRun(ctx context.Context, t target, run, next string) (bool, error) { req := &historyservice.DescribeMutableStateRequest{ @@ -187,8 +184,7 @@ func (h *handler) checkChain( } // checkActivity refuses an Activity's write whose task token isn't the Activity's current -// attempt. The epoch stays the Producer's, (attempt scheduled time, attempt), and its attempt -// must be the token's. +// attempt. The Producer's run, id and epoch attempt must match the token. func (h *handler) checkActivity( ctx context.Context, namespaceName string, diff --git a/chasm/lib/stream/producer.go b/chasm/lib/stream/producer.go index 7706dec9d47..03845d04c77 100644 --- a/chasm/lib/stream/producer.go +++ b/chasm/lib/stream/producer.go @@ -68,14 +68,13 @@ func (s *Stream) admit( } // newEpoch says whether a write is the first of a higher epoch of a Producer whose lower epoch -// wrote Records, so its Superseded Record withdraws them, as the crate's append script does. A -// Local Activity's next attempt in the same Workflow Task redoes the call. +// wrote Records, so its Superseded Record withdraws them. A Local Activity's retry redoes the call. func newEpoch(producer *streamservice.Producer, known *streampb.ProducerState) bool { return withdrawsEpochs(producer.GetKind()) && known != nil && known.GetHolds() != nil && CompareEpochs(known.GetHolds(), producer.GetEpoch()) < 0 } -// settle keeps the facts of a write that went in: the epoch, the task index and the run. +// settle keeps the task facts and the fenced run of a write that succeeded. func (s *Stream) settle( ctx chasm.MutableContext, producer *streamservice.Producer, @@ -108,8 +107,8 @@ func (s *Stream) settle( } } -// withdrawn says whether the Stream knows that a Superseded Record withdrew the Record: its run -// is fenced, its Workflow Task was withdrawn, or a higher epoch of its Producer wrote. +// withdrawn says whether the Stream knows that a Superseded Record withdrew the Record. It knows +// when the run is fenced, the Workflow Task was withdrawn, or a higher epoch of the Producer wrote. func (s *Stream) withdrawn(ctx chasm.Context, record *streamservice.StreamRecord) bool { producer := record.GetProducer() if s.runFenced(producer.GetRunId()) { @@ -129,9 +128,8 @@ func (s *Stream) withdrawn(ctx chasm.Context, record *streamservice.StreamRecord CompareEpochs(producer.GetEpoch(), known.GetEpoch()) < 0 } -// withdrawsEpochs says whether a higher epoch of a Producer outside Workflow code withdraws the -// Records of its lower epochs. An Activity's retry redoes the attempt's work. A client's higher -// epoch only fences the lower ones, since a client process writes on and doesn't redo. +// withdrawsEpochs says whether a non-Workflow Producer's higher epoch withdraws its lower epochs' +// Records. An Activity's retry redoes the attempt's work, but a client doesn't redo its writes. func withdrawsEpochs(kind streamservice.ProducerKind) bool { return kind != streamservice.PRODUCER_KIND_WORKFLOW && kind != streamservice.PRODUCER_KIND_CLIENT @@ -150,8 +148,8 @@ func (s *Stream) withdrawsRun(ctx chasm.Context, run string) bool { func supersededRecord(producer *streamservice.Producer, w Withdrawal) *streamservice.StreamRecord { epoch := producer.GetEpoch() if w.Defer { - // A deferred batch or a create withdraws only the retry run's previous run, so no epoch - // goes with it. + // A deferred batch or a create withdraws only the retry run's previous run, so its + // Superseded Record has no epoch. epoch = nil } return &streamservice.StreamRecord{ @@ -168,7 +166,7 @@ func supersededRecord(producer *streamservice.Producer, w Withdrawal) *streamser } } -// orphans applies to a Local Activity write with its run's last completed task (S110). It splits +// orphans applies to a Local Activity write that has its run's last completed task. It splits // the run's open Local Activity tasks into those that never completed and those still open. func (s *Stream) orphans( ctx chasm.Context, @@ -243,9 +241,8 @@ func completedTask(req *streamservice.AppendRecordsRequest) *streamservice.Workf return nil } -// orphanRecord withdraws a run's orphaned Local Activity Records. It names the run's Workflow -// Producer in the write's task, so a reader's Workflow rule withdraws the run's Local Activity -// Records of the tasks after the completed one and before that task, and of the failed ones. +// orphanRecord is a Superseded Record of the run's Workflow Producer at the write's task, so a +// reader's Workflow rule withdraws the run's orphaned Local Activity Records. func orphanRecord(req *streamservice.AppendRecordsRequest) *streamservice.StreamRecord { producer := req.GetProducer() epoch := producer.GetEpoch() @@ -266,7 +263,7 @@ func orphanRecord(req *streamservice.AppendRecordsRequest) *streamservice.Stream } } -// EncodeIDPart percent-encodes an id part as the crate does, so a Producer id matches its. +// EncodeIDPart percent-encodes an id part as the crate does, so a Producer id matches the crate's. func EncodeIDPart(text string) string { return encodePart(text) } @@ -289,10 +286,8 @@ type taskSplit struct { kept []*streamservice.WorkflowTaskEpoch } -// withdrawTasks is the crate's Workflow withdrawal rule over a task index. For task t with the -// previous completed task c, it withdraws the tasks in failed or strictly between c and t, and -// those at the started id of c or of t with a lower attempt. It keeps the tasks after c, since -// no task up to c can be withdrawn later. +// withdrawTasks is the crate's Workflow withdrawal rule for task t and the previous completed task +// c. The index keeps only the tasks after c, since no later write can withdraw a task up to c. func withdrawTasks( index []*streamservice.WorkflowTaskEpoch, t, c *streamservice.WorkflowTaskEpoch, diff --git a/chasm/lib/stream/proto/v1/message.proto b/chasm/lib/stream/proto/v1/message.proto index 17d5545c9f3..a9a2e937a37 100644 --- a/chasm/lib/stream/proto/v1/message.proto +++ b/chasm/lib/stream/proto/v1/message.proto @@ -129,7 +129,7 @@ message ParkedReadState { temporal.api.streamservice.v1.ReadRecordsRequest request = 1; // The Nexus request id, so a retry of the start finds this read. string request_id = 2; - // Where the next look starts. A look that finds only other Topics moves it on. + // Where the next look starts. A look that finds only other Topics advances it. int64 scan_offset = 3; // Set while the read waits for the create. google.protobuf.Timestamp create_deadline = 4; diff --git a/chasm/lib/stream/proto/v1/service.proto b/chasm/lib/stream/proto/v1/service.proto index 2ab7e88d358..3849baf0933 100644 --- a/chasm/lib/stream/proto/v1/service.proto +++ b/chasm/lib/stream/proto/v1/service.proto @@ -8,7 +8,7 @@ import "temporal/server/api/routing/v1/extension.proto"; option go_package = "go.temporal.io/server/chasm/lib/stream/gen/streampb;streampb"; -// StreamService carries the frontend's Stream calls to the history shard of the Stream. A Workflow +// StreamService serves the frontend's Stream calls on the history shard of the Stream. A Workflow // Stream is on its Workflow's shard, and a Standalone Activity's Stream on its Activity's. service StreamService { rpc CreateStream(CreateStreamRequest) returns (CreateStreamResponse) { diff --git a/chasm/lib/stream/read.go b/chasm/lib/stream/read.go index 60a4c976628..dd87fd08a83 100644 --- a/chasm/lib/stream/read.go +++ b/chasm/lib/stream/read.go @@ -46,9 +46,9 @@ func (s *Stream) StartOffset(req *streamservice.ReadRecordsRequest) (int64, erro return from, nil } -// Page reads the Records from the offset from. It is false when the page has no Record, so a -// reader waits. A page ends at the close Record, and a read past the close gives the close -// Record again, so every read of a closed Stream ends with it. +// Page reads the Records at or past from. Its bool is false when the page has no Record, so the +// reader waits. A read past the close gives the close Record again, so every read of a closed +// Stream ends with it. func (s *Stream) Page( ctx chasm.Context, req *streamservice.ReadRecordsRequest, @@ -81,7 +81,7 @@ func (s *Stream) Page( return resp, len(resp.GetRecords()) > 0, nil } -// scan collects the Records from from that pass the Topic filter, up to limit. +// scan collects the Records at or past from that pass the Topic filter, up to limit. func (s *Stream) scan( ctx chasm.Context, req *streamservice.ReadRecordsRequest, @@ -130,8 +130,8 @@ func (s *Stream) scan( return resp, nil } -// pageFull says whether a page ends before its next entry. Only a page with Records stops at the -// scan cap, so a reader never takes an empty page for the end with a match left. +// pageFull says whether a page ends before its next entry. An empty page doesn't stop at the scan +// cap, so a reader can't mistake it for the end while a match is left. func pageFull(resp *streamservice.ReadRecordsResponse, limit int, scanned int) bool { if len(resp.GetRecords()) == 0 { return false @@ -148,7 +148,7 @@ func (s *Stream) firstBatchAt(offset int64) int { } // overBytes says whether entry would take a page past its max_bytes. A page takes its first -// Record at any size, so a read always moves on. +// Record at any size, so a read always makes progress. func overBytes( req *streamservice.ReadRecordsRequest, resp *streamservice.ReadRecordsResponse, @@ -275,8 +275,8 @@ func passes(req *streamservice.ReadRecordsRequest, topic string) bool { return false } -// wakeAnswer is the answer of a wake_only read: no Records, the read's start as its next -// offset, and whether the Stream is closed. +// wakeAnswer is the answer of a wake_only read. It has no Records, and its next offset is the +// read's start. func (s *Stream) wakeAnswer(start int64) *streamservice.ReadRecordsResponse { return &streamservice.ReadRecordsResponse{ NextOffset: start, @@ -284,9 +284,8 @@ func (s *Stream) wakeAnswer(start int64) *streamservice.ReadRecordsResponse { } } -// afterClose gives the Superseded items written after the close from the offset from, and the -// offset after the last entry, zero if none. A Workflow Task that completes after the close -// still withdraws, and a reader stops at the close Record, so a page puts them before it. +// afterClose gives the Superseded items written after the close at or past from, and the offset +// after the last one. A reader stops at the close Record, so a page puts them before it. func (s *Stream) afterClose( ctx chasm.Context, from, end int64, @@ -451,8 +450,8 @@ type readOutcome struct { err error } -// wake answers a parked read when the Stream can. A look that finds only other Topics moves -// the read's offset on, so the next look starts after it. +// wake answers a parked read when the Stream can. A look that finds only other Topics advances +// the read's offset, so the next look starts after those entries. func (s *Stream) wake( ctx chasm.MutableContext, read *ParkedRead, @@ -467,7 +466,7 @@ func (s *Stream) wake( read.StartOffset = from read.ScanOffset = from } - // Nothing came since the read's last look, so every parked read doesn't rescan on each write. + // With no entry since the read's last look, a write doesn't rescan the parked read. if s.Phase == streampb.STREAM_PHASE_OPEN && read.GetScanOffset() > s.NewestOffset { return readOutcome{}, false } diff --git a/chasm/lib/stream/readers.go b/chasm/lib/stream/readers.go index ac6d80c5f24..c6f236689fd 100644 --- a/chasm/lib/stream/readers.go +++ b/chasm/lib/stream/readers.go @@ -25,7 +25,7 @@ type ReaderRun struct { } // readerRunOf finds the reading run in the links of a direct call. A call with no Workflow link, -// such as a replay's read again, keeps no floor. +// such as a read that a replay repeats, keeps no floor. func readerRunOf(links []nexus.Link) *ReaderRun { for _, link := range links { if event, err := commonnexus.ConvertNexusLinkToLinkWorkflowEvent(link); err == nil && @@ -36,8 +36,8 @@ func readerRunOf(links []nexus.Link) *ReaderRun { return nil } -// readerFloor is the floor that a read from from sets for the run, false when the run's floor is -// as low or the read fails anyway. Every Stream keeps floors, since its size cap trims too. +// readerFloor is the floor that a read at from sets for the run. It's false when the run's floor +// is as low already, or when the read fails anyway. func (s *Stream) readerFloor(ctx chasm.Context, runID string, from int64) (int64, bool) { if from == 0 { from = s.TrimmedOffset + 1 @@ -51,9 +51,8 @@ func (s *Stream) readerFloor(ctx chasm.Context, runID string, from int64) (int64 return from, true } -// floorStart is where a run's floor starts for a read from start. A reset run copied the reads of -// the old run into its History, and its first live read gives the lowest of them in floor_from, -// so the floor covers what its replay reads again. +// floorStart is where a run's floor starts for a read at start. A reset run's first live read +// gives the lowest offset of its copied reads in floor_from, so the floor covers its replay. func (s *Stream) floorStart(req *streamservice.ReadRecordsRequest, start int64) int64 { floorFrom := req.GetFloorFrom() if floorFrom <= 0 || start == 0 || floorFrom >= start { @@ -79,7 +78,7 @@ func (s *Stream) suggestsContinueAsNew(ctx chasm.Context, runID string, limits L return 2*kept > limits.MaxStoredBytes } -// AddReader keeps the floor of a Workflow run that reads from from, in the transaction that +// AddReader keeps the floor of a Workflow run that reads at from, in the transaction that // answers the read. No trim passes the floor while the run is open. func (s *Stream) AddReader( ctx chasm.MutableContext, @@ -196,7 +195,7 @@ func (h *readersCheckTaskHandler) Execute( } // runOpen says whether a Workflow run is still open. A deleted or unknown run is closed. The run -// may be on another shard, so it goes through the history client. +// may be on another shard, so the check calls the history client. func runOpen( ctx context.Context, historyClient resource.HistoryClient, diff --git a/chasm/lib/stream/state.go b/chasm/lib/stream/state.go index 4687423c14a..0274496fa96 100644 --- a/chasm/lib/stream/state.go +++ b/chasm/lib/stream/state.go @@ -105,8 +105,8 @@ func (s *Stream) dropReaders(ctx chasm.MutableContext, runIDs []string) { } } -// limitProducers drops Producers past the cap. A Local Activity call writes only while it runs, -// so its entry goes first. Then the Producer with the oldest last batch goes, with its fence. +// limitProducers drops Producers past the cap, each with its fence. A Local Activity call writes +// only while it runs, so its entry is dropped first, then the one with the oldest last batch. func (s *Stream) limitProducers(ctx chasm.MutableContext, maxProducers int) { localPrefix := streamservice.PRODUCER_KIND_LOCAL_ACTIVITY.String() + "|" for maxProducers > 0 && len(s.Producers) > maxProducers { @@ -134,8 +134,8 @@ func (s *Stream) oldestProducer(ctx chasm.MutableContext, match func(string) boo return oldestKey } -// DropActivityProducers drops the entries of an Activity of a run that closed. It never writes -// again, so its fence and dedupe state can go. +// DropActivityProducers drops the entries of an Activity of a run that closed. The Activity never +// writes again, so its fence and dedupe state aren't needed. func (s *Stream) DropActivityProducers(runID, activityID string) { prefix := streamservice.PRODUCER_KIND_ACTIVITY.String() + "|act:" + encodePart(runID) + ":" + encodePart(activityID) + ":" diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index 419abba6bf3..3969fb54465 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -195,8 +195,8 @@ func (s *Stream) reopen() { } } -// owned says whether a create's owner keeps the Stream open past one Retention with no write: a -// Workflow, or a Standalone Activity, which closes the Stream when it ends. +// owned says whether a create's owner keeps the Stream open past one Retention with no write. A +// Workflow and a Standalone Activity do, since each closes its Stream when it ends. func owned(owner *streamservice.Producer) bool { kind := owner.GetKind() return kind == streamservice.PRODUCER_KIND_WORKFLOW || @@ -211,7 +211,7 @@ type AppendInput struct { // Append writes one batch of one Producer, with the checks and the withdrawal of the crate's // append script in one transition. The same epoch and sequence answers the first offsets -// again, so a retried batch goes in once. +// again, so a retried batch is written once. func (s *Stream) Append( ctx chasm.MutableContext, in AppendInput, @@ -247,8 +247,8 @@ func (s *Stream) Append( "The sequence %d is below the last one, %d", req.GetSequence(), known.GetSequence()) } } - // A Workflow batch with no Records only settles its task facts, which a closed Stream - // still takes, as the Superseded Records after the close. + // A Workflow batch with no Records only settles its task facts. A closed Stream still takes + // it, since a Workflow Task that completes after the close can still withdraw Records. deferred := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && req.GetDeferWithdrawal() settles := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && @@ -338,8 +338,8 @@ func BatchDigest(records []*streamservice.AppendRecord) []byte { return h.Sum(nil) } -// batchEntries are the entries of one write: the orphans' Superseded Record, the batch's own -// Superseded Record, then the Records. +// batchEntries are the entries of one write. The orphans' Superseded Record comes first, then the +// batch's own Superseded Record, then the Records. func batchEntries( req *streamservice.AppendRecordsRequest, w Withdrawal, @@ -347,8 +347,8 @@ func batchEntries( ) []*streamservice.StreamRecord { producer := req.GetProducer() var entries []*streamservice.StreamRecord - // The orphans go in an entry of their own, since the batch's first entry may carry its own - // Superseded Record. + // The orphans get an entry of their own, since the batch may also write its own Superseded + // Record. if orphans { entries = append(entries, orphanRecord(req)) } @@ -770,7 +770,7 @@ func (s *Stream) nextDeadline(ctx chasm.Context, limits Limits) (time.Time, bool return deadline, found } -// finish wakes the reads that can answer now, drops the reads whose answer went out, applies +// finish wakes the reads that can answer now, drops the reads that have answered, applies // the size cap and arms the timer for the next deadline. func (s *Stream) finish(ctx chasm.MutableContext, limits Limits) { s.limitStored(limits.MaxStoredBytes) @@ -779,7 +779,7 @@ func (s *Stream) finish(ctx chasm.MutableContext, limits Limits) { return s.wake(ctx, read, limits) }) } else if s.waitingReads(ctx) == 0 { - // No read waits for the create any more, so the execution can go. + // No read waits for the create any more, so the Stream is done. s.Phase = streampb.STREAM_PHASE_ABANDONED } s.pruneReads(ctx) diff --git a/chasm/lib/stream/stream_test.go b/chasm/lib/stream/stream_test.go index 7d6421518b4..2c196ebffe9 100644 --- a/chasm/lib/stream/stream_test.go +++ b/chasm/lib/stream/stream_test.go @@ -768,8 +768,8 @@ func TestErrorsCarryTheReason(t *testing.T) { require.Equal(t, ReasonFenced, details[0].(*streamservice.StreamErrorDetails).GetReason()) } -// A reader's floor keeps its Records past the Retention and the cap, and the Records below it -// trim. When the reader goes, the Retention trims the rest. +// A reader's floor keeps its Records past the Retention and the cap, and the Records below it are +// trimmed. When the floor is dropped, the Retention trims the rest. func TestReaderFloorKeepsItsRecords(t *testing.T) { h := newHarness(t) h.limits.MaxStoredBytes = 200 @@ -884,8 +884,8 @@ func TestReaderFloorKeepsTheRecordsOfAClientStream(t *testing.T) { require.Equal(t, []string{"a"}, topics(page)) } -// localActivityWrite is a write of the Local Activity call of the run, dispatched by task -// started, with the run's last completed task (0 before the first), as the crate's S110 test has. +// localActivityWrite is a write of the run's Local Activity call from the task that started at +// started. It has the run's last completed task, or 0 before the first, as the crate's tests do. func localActivityWrite( call string, started, completed int64, @@ -1056,8 +1056,8 @@ func TestOnlyAWakeParks(t *testing.T) { require.Nil(t, h.parked("page")) } -// A page stops before the Record that would take it past max_bytes, and a larger Record comes -// alone, so a reader with a small cap still moves on. +// A page stops before the Record that would take it past max_bytes, and a larger Record is alone +// on its page, so a reader with a small cap still makes progress. func TestPageStopsAtMaxBytes(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) @@ -1228,7 +1228,7 @@ func TestReplayOfAPagePastTheCloseGetsTheClose(t *testing.T) { } // A client's higher epoch fences its lower epochs but withdraws none of their Records, since a -// client process writes on and doesn't redo the work. +// client process doesn't redo its writes. func TestClientEpochFencesButWithdrawsNothing(t *testing.T) { h := newHarness(t) _, err := h.create(&streamservice.CreateStreamRequest{StreamId: "chat-1"}) @@ -1271,8 +1271,8 @@ func TestRecordsOfOneAppendHaveConsecutiveOffsets(t *testing.T) { require.Equal(t, resp.GetFirstOffset()+2, resp.GetLastOffset()) } -// At the Producer cap a Local Activity's entry goes first, since a call writes only while it -// runs. A client's older entry keeps its dedupe state, so its resend still goes in once. +// At the Producer cap a Local Activity's entry is dropped first, since a call writes only while it +// runs. A client's older entry keeps its dedupe state, so its resend is still written once. func TestLocalActivityProducerGoesFirstAtTheCap(t *testing.T) { h := newHarness(t) h.limits.MaxProducers = 2 diff --git a/chasm/lib/workflow/workflow.go b/chasm/lib/workflow/workflow.go index f7a5c3e41cd..26bc7b7bc04 100644 --- a/chasm/lib/workflow/workflow.go +++ b/chasm/lib/workflow/workflow.go @@ -52,7 +52,7 @@ type Workflow struct { Updates chasm.Map[string, *WorkflowUpdate] // Streams are the native Streams of the run chain, keyed by Stream id. They nest in the - // chain's first run, since a run's tree doesn't move to the next run. + // chain's first run, since each run of a chain has its own tree. Streams chasm.Map[string, *stream.Stream] } @@ -118,8 +118,8 @@ func (w *Workflow) CheckWriter(chasm.Context, *streamservice.Producer, []byte) e return nil } -// HasChainStreams says whether the final close of this run may have Streams to close: its own, -// or, for a later run of a chain, the ones nested in the chain's first run. +// HasChainStreams says whether the final close of this run may have Streams to close. Those are +// its own, or, for a later run of a chain, the ones nested in the chain's first run. func (w *Workflow) HasChainStreams(ctx chasm.Context, firstRunID string) bool { if len(w.Streams) > 0 { return true @@ -161,7 +161,7 @@ func (w *Workflow) EndChainStreams( // DropActivityProducers drops a closed Activity's Producer entries from the Streams of this run. // After continue-as-new the Streams are in the chain's first run, another execution, and keep -// the entries until the Producer cap. +// the entries until the Producer cap drops them. func (w *Workflow) DropActivityProducers(ctx chasm.MutableContext, activityID string) { runID := ctx.ExecutionKey().RunID for _, field := range w.Streams { diff --git a/common/rpc/interceptor/namespace_validator.go b/common/rpc/interceptor/namespace_validator.go index 50dc45e2cc4..9b8ed698ba7 100644 --- a/common/rpc/interceptor/namespace_validator.go +++ b/common/rpc/interceptor/namespace_validator.go @@ -425,8 +425,8 @@ func (ni *NamespaceValidatorInterceptor) extractNamespaceFromTaskToken(req any) return ni.namespaceRegistry.GetNamespaceByID(namespaceID) } -// optionalTaskToken says whether a request carries a task token only from some callers. A Stream -// write carries one only from an Activity, and other callers set the namespace. +// optionalTaskToken says whether a request has a task token only from some callers. A Stream +// write has one only from an Activity, and other callers set the namespace. func optionalTaskToken(req any) bool { switch req.(type) { case *streamservice.CreateStreamRequest, diff --git a/service/frontend/nexus_direct.go b/service/frontend/nexus_direct.go index 2257376d76a..8769eec1e6e 100644 --- a/service/frontend/nexus_direct.go +++ b/service/frontend/nexus_direct.go @@ -17,8 +17,8 @@ import ( ) // StartNexusOperation runs one operation of the System Nexus endpoint for the caller's namespace, -// as a direct call (temporalio/api#720). It creates no caller-side state and no History event, -// so a Workflow can read through it with nothing in its History. +// as a direct call. It creates no caller-side state and no History event, so a Workflow can read +// through it with nothing in its History. func (wh *WorkflowHandler) StartNexusOperation( ctx context.Context, req *workflowservice.StartNexusOperationRequest, @@ -101,9 +101,8 @@ func (wh *WorkflowHandler) StartNexusOperation( } } -// workflowEventLinks passes on the caller's Workflow links, so an operation knows the reading run. -// A direct call names no event of its caller, and the Nexus form of a link needs a reference, so a -// link with none refers to the call's request id. +// workflowEventLinks gives an operation the caller's Workflow links, so it knows the reading run. +// The Nexus form of a link needs a reference, so a link with none refers to the call's request id. func workflowEventLinks(links []*commonpb.Link, requestID string) []*nexuspb.Link { var out []nexus.Link for _, link := range links { diff --git a/tests/stream_test.go b/tests/stream_test.go index ed1c3f74d81..289c1b26330 100644 --- a/tests/stream_test.go +++ b/tests/stream_test.go @@ -790,8 +790,8 @@ func (s *StreamTestSuite) TestCloseCommandOfAFailedWorkflowTaskIsDropped() { ) } -// A cron run's end isn't the chain's end, so the next cron run writes on. The terminate of a later -// run ends the chain and closes the Stream with its status. +// A cron run's end isn't the chain's end, so the next cron run still writes. The terminate of a +// later run ends the chain and closes the Stream with its status. func (s *StreamTestSuite) TestCronRunEndKeepsTheStreamOpen() { env := s.newEnv() sc := s.streamClient(env) @@ -985,7 +985,7 @@ func (s *StreamTestSuite) TestStreamClosesWithTheLastRunOfTheChain() { } // A reset to an earlier run of a continued-as-new chain terminates the current run, but the -// chain goes on in the reset run, so its Streams stay open. +// chain continues in the reset run, so its Streams stay open. func (s *StreamTestSuite) TestResetToAnEarlierRunKeepsTheStreamOpen() { env := s.newEnv() sc := s.streamClient(env) @@ -1355,7 +1355,7 @@ func (s *StreamTestSuite) TestWakeInAClosedFirstRunEndsAtItsWait() { } // The readers check runs in a first run that continued as new, so the floor of a reader that -// closed goes, and the Retention trims its range. +// closed is dropped, and the Retention trims its range. func (s *StreamTestSuite) TestReadersCheckRunsInAClosedFirstRun() { env := s.newEnv(testcore.WithDynamicConfig(stream.ReaderCheckInterval, time.Second)) sc := s.streamClient(env) @@ -1412,7 +1412,7 @@ func (s *StreamTestSuite) TestAppendToAStreamThatIsNotThereFails() { s.Empty(page.GetRecords()) } -// A Workflow Stream that no Workflow reads trims past the size cap, so its writes go on. +// A Workflow Stream that no Workflow reads trims past the size cap, so its writes don't fail. func (s *StreamTestSuite) TestWorkflowStreamWithNoReaderTrimsPastTheCap() { env := s.newEnv() sc := s.streamClient(env) From cbc55d7195a94162a3a8eeb460c756a6d71a66c1 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 04:23:55 -0700 Subject: [PATCH 21/23] Applied the go fix modernizers to the Stream reads. The fmt job runs go fix and fails on the loops it rewrites to slices.Backward and slices.Contains. --- chasm/lib/stream/read.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/chasm/lib/stream/read.go b/chasm/lib/stream/read.go index dd87fd08a83..344301fe99a 100644 --- a/chasm/lib/stream/read.go +++ b/chasm/lib/stream/read.go @@ -183,10 +183,9 @@ func (s *Stream) Last( var size int64 scanned := 0 collect := func() { - for i := len(s.Index) - 1; i >= 0; i-- { - entries := s.Batches[s.Index[i].GetFirstOffset()].Get(ctx).GetRecords() - for j := len(entries) - 1; j >= 0; j-- { - entry := entries[j] + for _, info := range slices.Backward(s.Index) { + entries := s.Batches[info.GetFirstOffset()].Get(ctx).GetRecords() + for _, entry := range slices.Backward(entries) { if entry.GetOffset() < retained || len(picked) >= want || scanned >= maxScan { return } @@ -262,10 +261,8 @@ func passes(req *streamservice.ReadRecordsRequest, topic string) bool { if len(req.GetTopics()) == 0 && len(req.GetTopicPrefixes()) == 0 { return true } - for _, t := range req.GetTopics() { - if t == topic { - return true - } + if slices.Contains(req.GetTopics(), topic) { + return true } for _, prefix := range req.GetTopicPrefixes() { if strings.HasPrefix(topic, prefix) { From 4d6cfea209b644626e1948cc22f66b6d6039062d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 04:23:55 -0700 Subject: [PATCH 22/23] Dropped the go.sum lines of the replaced go.temporal.io/api. The replace points the module at api-go, so go mod tidy drops the upstream sums that Misc checks flags. --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 079bfbc0bfc..c80ec64a793 100644 --- a/go.sum +++ b/go.sum @@ -481,8 +481,6 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= -go.temporal.io/api v1.63.8-0.20261014221631-25d6833c5701 h1:4FS8eouE2ml0soTg/pht5tk/tBT5OpI64DGHXpJ0uqo= -go.temporal.io/api v1.63.8-0.20261014221631-25d6833c5701/go.mod h1:acM0I9WPuYg8W3Pd9jOZvEgi7mRUttUQ4+e7fowKVnM= go.temporal.io/auto-scaled-workers v0.0.0-20261013231030-67ffdef6330d h1:cKo/dGcLpFAZXKxIAR2sg9hI+h8/8Z0tNHxXA7LCRVI= go.temporal.io/auto-scaled-workers v0.0.0-20261013231030-67ffdef6330d/go.mod h1:DpnPBMkwtk/08wMxwR3u3mIOshsgK/V7KKLh1QzNdjw= go.temporal.io/sdk v1.49.0 h1:CtGI0BUe/SCo3eoqTwuWWtXKueii9GBVus7KrKKH1Vo= From 9f912df373d1fbe15bf1a395d26db9ebdf9a72fa Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 04:24:18 -0700 Subject: [PATCH 23/23] Split ReadOrPark, Append and scan into helpers under revive's limits. The golangci job failed on their complexity. The helpers keep the behavior. --- chasm/lib/stream/read.go | 165 ++++++++++++++++++++++++------------- chasm/lib/stream/stream.go | 98 ++++++++++++++-------- 2 files changed, 169 insertions(+), 94 deletions(-) diff --git a/chasm/lib/stream/read.go b/chasm/lib/stream/read.go index 344301fe99a..e6f229155a1 100644 --- a/chasm/lib/stream/read.go +++ b/chasm/lib/stream/read.go @@ -1,6 +1,7 @@ package stream import ( + "iter" "slices" "sort" "strings" @@ -98,36 +99,47 @@ func (s *Stream) scan( resp := &streamservice.ReadRecordsResponse{NextOffset: from} scanned := 0 var size int64 - for _, info := range s.Index[s.firstBatchAt(start):] { - for _, entry := range s.Batches[info.GetFirstOffset()].Get(ctx).GetRecords() { - if entry.GetOffset() < start { - continue - } - if end := req.GetToOffset(); end > 0 && entry.GetOffset() >= end { - return resp, nil - } - if pageFull(resp, limit, scanned) { - return resp, nil - } - if entry.GetOffset() >= retained && overBytes(req, resp, entry, size) { - return resp, nil + for entry := range s.entriesFrom(ctx, start) { + if end := req.GetToOffset(); end > 0 && entry.GetOffset() >= end { + return resp, nil + } + if pageFull(resp, limit, scanned) { + return resp, nil + } + if entry.GetOffset() >= retained && overBytes(req, resp, entry, size) { + return resp, nil + } + scanned++ + resp.NextOffset = entry.GetOffset() + 1 + if entry.GetOffset() < retained { + if from > 0 { + return nil, expired() } - scanned++ - resp.NextOffset = entry.GetOffset() + 1 - if entry.GetOffset() < retained { - if from > 0 { - return nil, expired() + continue + } + taken := len(resp.Records) + s.take(ctx, req, resp, entry) + if len(resp.Records) > taken { + size += int64(proto.Size(entry)) + } + } + return resp, nil +} + +// entriesFrom gives the stored entries at or past offset, oldest first. +func (s *Stream) entriesFrom( + ctx chasm.Context, + offset int64, +) iter.Seq[*streamservice.StreamRecord] { + return func(yield func(*streamservice.StreamRecord) bool) { + for _, info := range s.Index[s.firstBatchAt(offset):] { + for _, entry := range s.Batches[info.GetFirstOffset()].Get(ctx).GetRecords() { + if entry.GetOffset() >= offset && !yield(entry) { + return } - continue - } - taken := len(resp.Records) - s.take(ctx, req, resp, entry) - if len(resp.Records) > taken { - size += int64(proto.Size(entry)) } } } - return resp, nil } // pageFull says whether a page ends before its next entry. An empty page doesn't stop at the scan @@ -356,15 +368,8 @@ func (s *Stream) ReadOrPark( in ReadInput, ) (*streamservice.ReadRecordsResponse, error) { s.applyDue(ctx, in.Limits) - if field, ok := s.Reads[in.RequestID]; ok { - read := field.Get(ctx) - if !read.GetResolved() { - return nil, nil - } - if read.GetFailure() != nil { - return nil, errorOfFailure(read.GetFailure()) - } - return common.CloneProto(read.GetResponse()), nil + if resp, known, err := s.parkedAnswer(ctx, in.RequestID); known { + return resp, err } now := ctx.Now(s) read := &ParkedRead{ @@ -388,32 +393,74 @@ func (s *Stream) ReadOrPark( return nil, notFound() } read.CreateDeadline = timestamppb.New(now.Add(createWait)) - } else { - from, err := s.StartOffset(in.Request) - if err != nil { - return nil, err - } - if err := s.keepWakeFloor(ctx, in, from); err != nil { - return nil, err - } - resp, found, err := s.Page(ctx, in.Request, from) - if err != nil { - return nil, err - } - if found || !parks { - s.finish(ctx, in.Limits) - if in.Request.GetWakeOnly() { - resp = s.wakeAnswer(from) - } - resp.ContinueAsNewSuggested = s.suggestsContinueAsNew(ctx, read.GetReaderRunId(), - in.Limits) - return resp, nil + } else if resp, err := s.firstLook(ctx, in, read); resp != nil || err != nil { + return resp, err + } + return nil, s.park(ctx, in, read, now) +} + +// parkedAnswer answers a retry of a read's start from the read that it parked. It is false when +// the Stream has no read of the request id. +func (s *Stream) parkedAnswer( + ctx chasm.Context, + requestID string, +) (*streamservice.ReadRecordsResponse, bool, error) { + field, ok := s.Reads[requestID] + if !ok { + return nil, false, nil + } + read := field.Get(ctx) + switch { + case !read.GetResolved(): + return nil, true, nil + case read.GetFailure() != nil: + return nil, true, errorOfFailure(read.GetFailure()) + default: + return common.CloneProto(read.GetResponse()), true, nil + } +} + +// firstLook pages a read of a Stream that exists. It answers the page when the read finds Records +// or doesn't park, and otherwise sets where the parked read looks next. +func (s *Stream) firstLook( + ctx chasm.MutableContext, + in ReadInput, + read *ParkedRead, +) (*streamservice.ReadRecordsResponse, error) { + from, err := s.StartOffset(in.Request) + if err != nil { + return nil, err + } + if err := s.keepWakeFloor(ctx, in, from); err != nil { + return nil, err + } + resp, found, err := s.Page(ctx, in.Request, from) + if err != nil { + return nil, err + } + if found || !in.parks() { + s.finish(ctx, in.Limits) + if in.Request.GetWakeOnly() { + resp = s.wakeAnswer(from) } - read.StartOffset = from - read.ScanOffset = max(from, resp.GetNextOffset()) + resp.ContinueAsNewSuggested = s.suggestsContinueAsNew(ctx, read.GetReaderRunId(), + in.Limits) + return resp, nil } + read.StartOffset = from + read.ScanOffset = max(from, resp.GetNextOffset()) + return nil, nil +} + +// park keeps the read with its completion callback until the Stream can answer it. +func (s *Stream) park( + ctx chasm.MutableContext, + in ReadInput, + read *ParkedRead, + now time.Time, +) error { if s.waitingReads(ctx) >= in.Limits.MaxParkedReads { - return nil, serviceerror.NewResourceExhausted( + return serviceerror.NewResourceExhausted( enumspb.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT, "The Stream has the most waiting reads") } @@ -424,10 +471,10 @@ func (s *Stream) ReadOrPark( callbacks := []*commonpb.Callback{in.Callback} err := callback.Attach(ctx, read, in.RequestID, timestamppb.New(now), callbacks) if err != nil { - return nil, err + return err } s.finish(ctx, in.Limits) - return nil, nil + return nil } // CancelRead completes a parked read as canceled. diff --git a/chasm/lib/stream/stream.go b/chasm/lib/stream/stream.go index 3969fb54465..6e660344ef2 100644 --- a/chasm/lib/stream/stream.go +++ b/chasm/lib/stream/stream.go @@ -230,37 +230,17 @@ func (s *Stream) Append( } key := producerKey(producer) digest := BatchDigest(req.GetRecords()) - known := s.producer(ctx, key) - if known != nil && CompareEpochs(producer.GetEpoch(), known.GetEpoch()) == 0 { - if req.GetSequence() == known.GetSequence() { - if len(known.GetDigest()) > 0 && !bytes.Equal(known.GetDigest(), digest) { - return nil, NewError(ReasonSeqDivergent, - "The sequence %d has other content than its batch", req.GetSequence()) - } - return &streamservice.AppendRecordsResponse{ - FirstOffset: known.GetFirstOffset(), - LastOffset: known.GetLastOffset(), - }, nil - } - if req.GetSequence() < known.GetSequence() { - return nil, NewError(ReasonSequence, - "The sequence %d is below the last one, %d", req.GetSequence(), known.GetSequence()) - } + if resp, err := resend(req, s.producer(ctx, key), digest); resp != nil || err != nil { + return resp, err } + w := withdrawalOf(req) // A Workflow batch with no Records only settles its task facts. A closed Stream still takes // it, since a Workflow Task that completes after the close can still withdraw Records. - deferred := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && - req.GetDeferWithdrawal() settles := producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW && - len(req.GetRecords()) == 0 && - !deferred + len(req.GetRecords()) == 0 && !w.Defer if s.Phase == streampb.STREAM_PHASE_CLOSED && !settles { return nil, NewError(ReasonClosed, "The Stream is closed") } - w := Withdrawal{WithdrawRun: req.GetWithdrawRun(), Defer: deferred} - if producer.GetKind() == streamservice.PRODUCER_KIND_WORKFLOW { - w.PreviousCompleted, w.FailedTasks = req.GetPreviousCompletedTask(), req.GetFailedTasks() - } flagged, err := s.admit(ctx, producer, w) if err != nil { return nil, err @@ -285,22 +265,14 @@ func (s *Stream) Append( if orphans { s.settleOrphans(ctx, producer, orphaned, wrote) } - state := &streampb.ProducerState{ + s.setProducer(ctx, key, &streampb.ProducerState{ Epoch: producer.GetEpoch(), Sequence: req.GetSequence(), FirstOffset: first, LastOffset: last, - Holds: s.producer(ctx, key).GetHolds(), + Holds: newHolds(producer, s.producer(ctx, key).GetHolds(), wrote, flagged), Digest: digest, - } - if withdrawsEpochs(producer.GetKind()) { - if wrote { - state.Holds = producer.GetEpoch() - } else if flagged { - state.Holds = nil - } - } - s.setProducer(ctx, key, state) + }) if wrote { s.LastWriteTime = timestamppb.New(ctx.Now(s)) } @@ -309,6 +281,62 @@ func (s *Stream) Append( return &streamservice.AppendRecordsResponse{FirstOffset: first, LastOffset: last}, nil } +// resend answers a resend of a Producer's last batch with that batch's offsets, and refuses a +// lower sequence or other content. It answers nil for a new batch. +func resend( + req *streamservice.AppendRecordsRequest, + known *streampb.ProducerState, + digest []byte, +) (*streamservice.AppendRecordsResponse, error) { + if known == nil || CompareEpochs(req.GetProducer().GetEpoch(), known.GetEpoch()) != 0 { + return nil, nil + } + switch { + case req.GetSequence() > known.GetSequence(): + return nil, nil + case req.GetSequence() < known.GetSequence(): + return nil, NewError(ReasonSequence, + "The sequence %d is below the last one, %d", req.GetSequence(), known.GetSequence()) + case len(known.GetDigest()) > 0 && !bytes.Equal(known.GetDigest(), digest): + return nil, NewError(ReasonSeqDivergent, + "The sequence %d has other content than its batch", req.GetSequence()) + default: + return &streamservice.AppendRecordsResponse{ + FirstOffset: known.GetFirstOffset(), + LastOffset: known.GetLastOffset(), + }, nil + } +} + +// withdrawalOf is what a batch withdraws. Only a Workflow batch knows its Workflow Task facts. +func withdrawalOf(req *streamservice.AppendRecordsRequest) Withdrawal { + w := Withdrawal{WithdrawRun: req.GetWithdrawRun()} + if req.GetProducer().GetKind() == streamservice.PRODUCER_KIND_WORKFLOW { + w.Defer = req.GetDeferWithdrawal() + w.PreviousCompleted, w.FailedTasks = req.GetPreviousCompletedTask(), req.GetFailedTasks() + } + return w +} + +// newHolds is the epoch whose Records the Producer's next higher epoch withdraws. A batch with +// Records holds its own epoch, and a batch that withdrew the held Records holds none. +func newHolds( + producer *streamservice.Producer, + held *streamservice.ProducerEpoch, + wrote, flagged bool, +) *streamservice.ProducerEpoch { + switch { + case !withdrawsEpochs(producer.GetKind()): + return held + case wrote: + return producer.GetEpoch() + case flagged: + return nil + default: + return held + } +} + // BatchDigest is the SHA-256 of a batch's content, as the crate and the memory Store compute it. // Each Record gives its Topic, its metadata count, its metadata sorted by key, and its data, each // part with a big-endian u64 length first. A resend with the same sequence must have the same