From 270c019e0b4f3123c808d29377ef6e921ad19bd2 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Wed, 28 Oct 2026 07:45:40 +0200 Subject: [PATCH] feat(task): add resumable retry with operator decision hook Retries can now pause for an operator decision through RetryConfig.OnError, which can resume the same operation with a message, fail with the original cause, or follow the automatic policy. Retry behavior is unchanged when OnError is nil. Pending requests are exposed in TaskSnapshot.Retry while the host waits, and cancellation of the waiting task is handled explicitly. --- task/doc.go | 7 +++ task/example_retry_resume_test.go | 26 ++++++++++ task/retry_resume.go | 79 +++++++++++++++++++++++++++++ task/retry_resume_test.go | 82 +++++++++++++++++++++++++++++++ task/snapshot.go | 5 ++ task/task.go | 11 +++-- task/task_types.go | 18 +++---- task/worker.go | 49 +++++++++--------- 8 files changed, 240 insertions(+), 37 deletions(-) create mode 100644 task/example_retry_resume_test.go create mode 100644 task/retry_resume.go create mode 100644 task/retry_resume_test.go diff --git a/task/doc.go b/task/doc.go index a7932762..dafd0064 100644 --- a/task/doc.go +++ b/task/doc.go @@ -233,6 +233,13 @@ policy to code that is not a task: return indexModule(ctx) }) +RetryConfig.OnError receives the original error and remaining automatic retry budget. +The host can await user input, honoring the callback context, and return a RetryDecision. +RetryResume resumes the operation even after MaxRetries; RetryFail preserves the error; +RetryAutomatic follows the bounded retry policy. RetryMessage(ctx) exposes the user's +message to the resumed attempt. Pending requests appear in TaskSnapshot.Retry until +the decision or cancellation. See ExampleRetryMessage for a runnable example. + # Fatal Errors For unrecoverable errors that should stop execution: diff --git a/task/example_retry_resume_test.go b/task/example_retry_resume_test.go new file mode 100644 index 00000000..85c316dd --- /dev/null +++ b/task/example_retry_resume_test.go @@ -0,0 +1,26 @@ +package task_test + +import ( + "context" + "errors" + "fmt" + "github.com/flanksource/clicky/task" +) + +func ExampleRetryMessage() { + config := task.RetryConfig{OnError: func(ctx context.Context, request task.RetryRequest) (task.RetryDecision, error) { + // A terminal, approval broker, or web host can await the operator here. + return task.RetryDecision{Action: task.RetryResume, Message: "use the corrected configuration"}, nil + }} + err := task.Retry(context.Background(), config, func(ctx context.Context, attempt int) error { + if attempt == 1 { + return errors.New("invalid configuration") + } + fmt.Println(task.RetryMessage(ctx)) + return nil + }) + fmt.Println(err) + // Output: + // use the corrected configuration + // +} diff --git a/task/retry_resume.go b/task/retry_resume.go new file mode 100644 index 00000000..d4a33c2f --- /dev/null +++ b/task/retry_resume.go @@ -0,0 +1,79 @@ +package task + +import ( + "context" + "errors" + "fmt" + "time" +) + +type RetryAction string + +// RetryResume can resume after MaxRetries; RetryAutomatic keeps the bounded policy. +const ( + RetryAutomatic RetryAction = "automatic" + RetryResume RetryAction = "retry" + RetryFail RetryAction = "fail" +) + +type RetryRequest struct { + Attempt int `json:"attempt"` + Error string `json:"error"` + Automatic bool `json:"automatic"` + RemainingRetries int `json:"remainingRetries"` + Cause error `json:"-"` +} + +type RetryDecision struct { + Action RetryAction `json:"action"` + Message string `json:"message,omitempty"` +} + +type retryMessageKey struct{} + +// RetryMessage is the user's message for the resumed attempt. It is never +// logged or included in snapshots; the operation decides how to use it. +func RetryMessage(ctx context.Context) string { + message, _ := ctx.Value(retryMessageKey{}).(string) + return message +} + +func retryAfterError(ctx context.Context, config RetryConfig, attempt int, cause error) (context.Context, bool, error) { + if ctx.Err() != nil { + return ctx, false, errors.Join(cause, context.Cause(ctx)) + } + retryable := shouldRetryError(cause, config) + automatic := retryable && attempt <= config.MaxRetries + if config.OnError != nil { + decision, err := config.OnError(ctx, RetryRequest{Attempt: attempt, Error: cause.Error(), Cause: cause, Automatic: automatic, RemainingRetries: max(0, config.MaxRetries-attempt+1)}) + if err != nil { + return ctx, false, errors.Join(cause, fmt.Errorf("retry decision: %w", err)) + } + if ctx.Err() != nil { + return ctx, false, errors.Join(cause, context.Cause(ctx)) + } + switch decision.Action { + case RetryResume: + return context.WithValue(ctx, retryMessageKey{}, decision.Message), true, nil + case RetryFail: + return ctx, false, cause + case RetryAutomatic: + default: + return ctx, false, errors.Join(cause, fmt.Errorf("invalid retry decision %q", decision.Action)) + } + } + if !automatic { + if retryable && attempt > config.MaxRetries { + cause = fmt.Errorf("failed after %d attempts: %w", attempt, cause) + } + return ctx, false, cause + } + timer := time.NewTimer(config.Delay(attempt)) + defer timer.Stop() + select { + case <-timer.C: + return ctx, true, nil + case <-ctx.Done(): + return ctx, false, errors.Join(cause, context.Cause(ctx)) + } +} diff --git a/task/retry_resume_test.go b/task/retry_resume_test.go new file mode 100644 index 00000000..1301b6e4 --- /dev/null +++ b/task/retry_resume_test.go @@ -0,0 +1,82 @@ +package task_test + +import ( + "context" + "errors" + "sync/atomic" + + "github.com/flanksource/clicky/task" + commonscontext "github.com/flanksource/commons/context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Resumable retry", func() { + It("exposes the cause and passes user input to a resumed attempt after automatic retries are exhausted", func() { + cause := errors.New("the selected configuration is invalid") + message := "The configuration has been corrected; use the new settings." + var requests []task.RetryRequest + config := task.RetryConfig{MaxRetries: 0, OnError: func(_ context.Context, request task.RetryRequest) (task.RetryDecision, error) { + requests = append(requests, request) + return task.RetryDecision{Action: task.RetryResume, Message: message}, nil + }} + calls := 0 + err := task.Retry(context.Background(), config, func(ctx context.Context, attempt int) error { + calls++ + if attempt == 1 { + return cause + } + Expect(task.RetryMessage(ctx)).To(Equal(message)) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(calls).To(Equal(2)) + Expect(requests).To(HaveLen(1)) + Expect(errors.Is(requests[0].Cause, cause)).To(BeTrue()) + Expect(requests[0].Error).To(Equal(cause.Error())) + Expect(requests[0].Automatic).To(BeFalse()) + }) + It("uses the same response seam for tracked tasks", func() { + var calls atomic.Int32 + config := task.RetryConfig{OnError: func(context.Context, task.RetryRequest) (task.RetryDecision, error) { + return task.RetryDecision{Action: task.RetryResume, Message: "continue"}, nil + }} + handle := task.StartTask("Resumable operation", func(ctx commonscontext.Context, _ *task.Task) (string, error) { + if calls.Add(1) == 1 { + return "", errors.New("operator input required") + } + return task.RetryMessage(ctx), nil + }, task.WithRetryConfig(config)) + result, err := handle.GetResult() + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("continue")) + Expect(calls.Load()).To(Equal(int32(2))) + }) + It("preserves the original cause when the operator declines to retry", func() { + cause := errors.New("invalid project settings") + config := task.RetryConfig{OnError: func(context.Context, task.RetryRequest) (task.RetryDecision, error) { + return task.RetryDecision{Action: task.RetryFail}, nil + }} + Expect(task.Retry(context.Background(), config, func(context.Context, int) error { return cause })).To(MatchError(cause)) + }) + It("publishes the pending error and cancels a task waiting for an operator", func() { + waiting := make(chan struct{}) + cause := errors.New("credentials need attention") + config := task.RetryConfig{OnError: func(ctx context.Context, request task.RetryRequest) (task.RetryDecision, error) { + close(waiting) + <-ctx.Done() + return task.RetryDecision{}, context.Cause(ctx) + }} + handle := task.StartTask("Waiting operation", func(commonscontext.Context, *task.Task) (bool, error) { return false, cause }, task.WithRetryConfig(config), task.WithCancellationDrain()) + Eventually(waiting).Should(BeClosed()) + snapshot := task.SnapshotTask(handle.GetTask(), nil) + Expect(snapshot.Error).To(Equal(cause.Error())) + Expect(snapshot.Retry).NotTo(BeNil()) + Expect(snapshot.Retry.Error).To(Equal(cause.Error())) + handle.Cancel() + _, err := handle.GetResult() + Expect(err).To(HaveOccurred()) + Expect(handle.GetTask().Status()).To(Equal(task.StatusCancelled)) + Expect(task.SnapshotTask(handle.GetTask(), nil).Retry).To(BeNil()) + }) +}) diff --git a/task/snapshot.go b/task/snapshot.go index 48e596aa..b4f4007a 100644 --- a/task/snapshot.go +++ b/task/snapshot.go @@ -37,6 +37,7 @@ type TaskSnapshot struct { Status string `json:"status"` Duration string `json:"duration,omitempty"` Error string `json:"error,omitempty"` + Retry *RetryRequest `json:"retry,omitempty"` Message string `json:"message,omitempty"` // latest log line Logs []LogEntry `json:"logs,omitempty"` // all log entries Total int `json:"total,omitempty"` // group: total child tasks @@ -85,6 +86,10 @@ func SnapshotTask(t *Task, group *Group) TaskSnapshot { Status: string(t.Status()), } t.mu.Lock() + if t.retryRequest != nil && t.status == StatusRunning { + request := *t.retryRequest + snap.Retry = &request + } dependencies := append([]*Task(nil), t.dependencies...) t.mu.Unlock() for _, dependency := range dependencies { diff --git a/task/task.go b/task/task.go index 2c1c50ec..d1031a07 100644 --- a/task/task.go +++ b/task/task.go @@ -42,11 +42,12 @@ type Task struct { bufferedLogger *logger.BufferedLogger // Structs - mu sync.Mutex - doneOnce sync.Once // Ensure done channel is closed only once - drainedOnce sync.Once // Ensure drained channel is closed only once - loggerOnce sync.Once // Ensure bufferedLogger is initialized only once - retryConfig RetryConfig + mu sync.Mutex + doneOnce sync.Once // Ensure done channel is closed only once + drainedOnce sync.Once // Ensure drained channel is closed only once + loggerOnce sync.Once // Ensure bufferedLogger is initialized only once + retryConfig RetryConfig + retryRequest *RetryRequest // 8-byte aligned types startTime time.Time diff --git a/task/task_types.go b/task/task_types.go index 5f072443..21fc41f1 100644 --- a/task/task_types.go +++ b/task/task_types.go @@ -122,6 +122,9 @@ type WaitResult struct { // RetryConfig holds configuration for task retry behavior type RetryConfig struct { + // OnError exposes a failed attempt to its host. The host may wait for a user + // decision, then resume the same operation with an optional message. + OnError func(context.Context, RetryRequest) (RetryDecision, error) RetryableErrors []string // Error message patterns that should trigger retries // Retryable, when set, decides retryability on its own (e.g. errors.Is // against a sentinel); RetryableErrors is then not consulted. @@ -164,18 +167,13 @@ func Retry(ctx context.Context, config RetryConfig, run func(ctx context.Context return context.Cause(ctx) } err := run(ctx, attempt) - if err == nil || !shouldRetryError(err, config) { + if err == nil { return err } - if attempt > config.MaxRetries { - return fmt.Errorf("failed after %d attempts: %w", attempt, err) - } - timer := time.NewTimer(config.Delay(attempt)) - select { - case <-timer.C: - case <-ctx.Done(): - timer.Stop() - return fmt.Errorf("retry cancelled after attempt %d (%v): %w", attempt, err, context.Cause(ctx)) + var retry bool + ctx, retry, err = retryAfterError(ctx, config, attempt, err) + if !retry { + return err } } } diff --git a/task/worker.go b/task/worker.go index ca214271..4a596a9c 100644 --- a/task/worker.go +++ b/task/worker.go @@ -215,6 +215,17 @@ func (t *Task) armTimeout() context.CancelFunc { // executeWithRetry handles task execution with exponential backoff retry func (w *worker) executeWithRetry(task *Task) { + config := task.retryConfig + if handler := config.OnError; handler != nil { + config.OnError = func(ctx context.Context, request RetryRequest) (RetryDecision, error) { + task.mu.Lock() + task.retryRequest = &request + task.mu.Unlock() + task.dirty.Store(true) + defer func() { task.mu.Lock(); task.retryRequest = nil; task.mu.Unlock(); task.dirty.Store(true) }() + return handler(ctx, request) + } + } for { // Check if task has a function to run if task.runFunc == nil { @@ -239,32 +250,26 @@ func (w *worker) executeWithRetry(task *Task) { } if err != nil { - // Check if error is retryable - shouldRetry := shouldRetryError(err, task.retryConfig) - - if shouldRetry && task.retryCount < task.retryConfig.MaxRetries { + task.dirty.Store(true) + next, retry, retryErr := retryAfterError(task.flanksourceCtx, config, task.retryCount+1, err) + if retry { + task.mu.Lock() task.retryCount++ - // Log retry attempt to bufferedLogger + task.flanksourceCtx.Context = next + task.ctx = task.flanksourceCtx + task.mu.Unlock() task.getBufferedLogger().Warnf("Attempt %d failed, retrying: %v", task.retryCount, err) - - // Calculate backoff delay - delay := task.retryConfig.Delay(task.retryCount) - - // Wait for delay or cancellation - select { - case <-time.After(delay): - continue // Retry - case <-task.ctx.Done(): - task.Cancel() - return - } - } else { - if _, failErr := task.FailedWithError(err); failErr != nil { - // Log error but continue - task is already in failed state - task.getBufferedLogger().Errorf("Failed to set error status: %v", failErr) - } + continue + } + if task.ctx.Err() != nil { + task.Cancel() return } + task.mu.Lock() + task.err = retryErr + task.mu.Unlock() + _, _ = task.FailedWithError(retryErr) + return } else { task.Success() return