From ddc23be26fe0d4c5ad05cc1984f79c258541954e Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:42:21 -0700 Subject: [PATCH 1/6] Return dropped search-attribute registration error in ebb_and_flow spawnWorkflowWithActivities ignored the error from RegisterDefaultSearchAttributes, so a registration failure was silently lost and surfaced later as an obscure workflow-start failure. --- scenarios/ebb_and_flow.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scenarios/ebb_and_flow.go b/scenarios/ebb_and_flow.go index a26451e6..c2e7f8b0 100644 --- a/scenarios/ebb_and_flow.go +++ b/scenarios/ebb_and_flow.go @@ -230,7 +230,9 @@ func (e *ebbAndFlowExecutor) Run(ctx context.Context, info loadgen.ScenarioInfo) config.MinBacklog, config.MaxBacklog, config.Period, e.Configuration.Duration) } - e.RegisterDefaultSearchAttributes(ctx) + if err := e.RegisterDefaultSearchAttributes(ctx); err != nil { + return fmt.Errorf("failed to register search attributes: %w", err) + } var started, completed, backlog, target int64 lastIteration := time.Now() From 7e1373930d162b631b5c42a6c909aeaebc16a589 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:44:06 -0700 Subject: [PATCH 2/6] Surface failed ebbAndFlowTrack workflows instead of swallowing them spawnWorkflowWithActivities logged wf.Get failures and returned nil, so they never reached the MaxConsecutiveErrors guard and a failed workflow was still counted as completed (inflating TotalCompletedWorkflows, which post-scenario verification relies on). Return the error and gate the completion count on success; the backlog still drains either way so the controller's behavior is unchanged. --- scenarios/ebb_and_flow.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scenarios/ebb_and_flow.go b/scenarios/ebb_and_flow.go index c2e7f8b0..f3449f93 100644 --- a/scenarios/ebb_and_flow.go +++ b/scenarios/ebb_and_flow.go @@ -376,10 +376,12 @@ func (e *ebbAndFlowExecutor) spawnWorkflowWithActivities( // Wait for workflow completion var result ebbandflow.WorkflowOutput err = wf.Get(ctx, &result) + // The batch has settled either way, so drain it from the backlog the + // controller tracks; only a successful workflow counts as completed. + e.completedActivities.Add(activities) if err != nil { - e.Logger.Errorf("ebbAndFlowTrack workflow failed for iteration %d: %v", iteration, err) + return fmt.Errorf("ebbAndFlowTrack workflow failed for iteration %d: %w", iteration, err) } - e.completedActivities.Add(activities) e.incrementTotalCompletedWorkflow() return nil From 80d99f3d9cb7906ccbb02a350b816fd3b54c2a88 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:44:40 -0700 Subject: [PATCH 3/6] Fix dropped terminate error and pointer log in ExecuteKitchenSinkWorkflow The client-actions goroutine logged the atomic.Pointer rather than the error value, and silently dropped the result of the cleanup TerminateWorkflow call. Log the actual error, and log the terminate failure (the goroutine has no return path for it). --- loadgen/scenario.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/loadgen/scenario.go b/loadgen/scenario.go index b02312da..0d50fc84 100644 --- a/loadgen/scenario.go +++ b/loadgen/scenario.go @@ -553,14 +553,13 @@ func (r *Run) ExecuteKitchenSinkWorkflow(ctx context.Context, options *KitchenSi err := executor.ExecuteClientSequence(cancelCtx, clientSeq) if err != nil { clientActionsErrPtr.Store(&err) - r.Logger.Error("Client actions failed: ", clientActionsErrPtr) + r.Logger.Error("Client actions failed: ", err) cancel() // TODO: Remove or change to "always terminate when exiting early" flag - err := r.Client.TerminateWorkflow( - ctx, options.StartOptions.ID, "", "client actions failed", nil) - if err != nil { - return + if terminateErr := r.Client.TerminateWorkflow( + ctx, options.StartOptions.ID, "", "client actions failed", nil); terminateErr != nil { + r.Logger.Errorf("Failed to terminate workflow after client actions failure: %v", terminateErr) } } }() From 6355d9db5673574cc0154fd097590b3e71fd857a Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:46:31 -0700 Subject: [PATCH 4/6] Stop double-handling terminal iteration errors in generic_executor The per-iteration goroutine logged the terminal failure and also sent it on doneCh, where Run returns it wrapped for its caller to report. Drop the goroutine log for the terminal branch (propagate only); keep the retry-branch log, whose transient error is superseded and never returned. --- loadgen/generic_executor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/loadgen/generic_executor.go b/loadgen/generic_executor.go index 3e656c2b..9b3d786b 100644 --- a/loadgen/generic_executor.go +++ b/loadgen/generic_executor.go @@ -187,11 +187,14 @@ func (g *genericRun) Run(ctx context.Context) error { backoff, retry := run.ShouldRetry(err) if retry { + // Transient: this attempt is superseded by the retry and + // never propagated, so log it here as the only record. err = fmt.Errorf("iteration %v encountered error: %w", run.Iteration, err) g.logger.Error(err) } else { + // Terminal: propagated via doneCh and reported by Run's + // caller, so don't also log it here. err = fmt.Errorf("iteration %v failed: %w", run.Iteration, err) - g.logger.Error(err) break retryLoop } From 2e88a3550bdcd17ba558218be9cf41ce0f13c76a Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:47:09 -0700 Subject: [PATCH 5/6] Branch on gRPC status code, not error string, in RegisterDefaultSearchAttributes Replace strings.Contains(err.Error(), ...) with status.Code(err) == codes.AlreadyExists, matching ensureNexusEndpoint in the same file. The second match string was a typo ("unavailble") that never matched, so no working behavior is lost. --- loadgen/scenario.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/loadgen/scenario.go b/loadgen/scenario.go index 0d50fc84..333d6409 100644 --- a/loadgen/scenario.go +++ b/loadgen/scenario.go @@ -410,22 +410,10 @@ func (s *ScenarioInfo) RegisterDefaultSearchAttributes(ctx context.Context) erro }, Namespace: s.Namespace, }) - // Throw an error if the attributes could not be registered, but ignore already exists errs - alreadyExistsStrings := []string{ - "already exists", - "attributes mapping unavailble", - } - if err != nil { - isAlreadyExistsErr := false - for _, s := range alreadyExistsStrings { - if strings.Contains(err.Error(), s) { - isAlreadyExistsErr = true - break - } - } - if !isAlreadyExistsErr { - return fmt.Errorf("failed to register search attributes: %w", err) - } + // Throw an error if the attributes could not be registered, but ignore the + // case where they already exist (re-registration on a reused namespace). + if err != nil && status.Code(err) != codes.AlreadyExists { + return fmt.Errorf("failed to register search attributes: %w", err) } return nil } From 93ad945ebb481b4042bb1852bea8428649af40c2 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:47:23 -0700 Subject: [PATCH 6/6] Wrap cert-load error with %w in loadTLSConfig Preserve the error chain at this library boundary with %w, matching the sibling wraps in the same function (lines 93, 118). --- clioptions/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clioptions/client.go b/clioptions/client.go index 9bfbd05d..fd2c2a2c 100644 --- a/clioptions/client.go +++ b/clioptions/client.go @@ -64,7 +64,7 @@ func (c *ClientOptions) loadTLSConfig() (*tls.Config, error) { } cert, err := tls.LoadX509KeyPair(c.ClientCertPath, c.ClientKeyPath) if err != nil { - return nil, fmt.Errorf("failed to load certs: %s", err) + return nil, fmt.Errorf("failed to load certs: %w", err) } tlsConfig.Certificates = []tls.Certificate{cert} return tlsConfig, nil