diff --git a/task/doc.go b/task/doc.go index a7932762..dafd0064 100644 --- a/task/doc.go +++ b/task/doc.go @@ -233,6 +233,13 @@ policy to code that is not a task: return indexModule(ctx) }) +RetryConfig.OnError receives the original error and remaining automatic retry budget. +The host can await user input, honoring the callback context, and return a RetryDecision. +RetryResume resumes the operation even after MaxRetries; RetryFail preserves the error; +RetryAutomatic follows the bounded retry policy. RetryMessage(ctx) exposes the user's +message to the resumed attempt. Pending requests appear in TaskSnapshot.Retry until +the decision or cancellation. See ExampleRetryMessage for a runnable example. + # Fatal Errors For unrecoverable errors that should stop execution: diff --git a/task/example_retry_resume_test.go b/task/example_retry_resume_test.go new file mode 100644 index 00000000..85c316dd --- /dev/null +++ b/task/example_retry_resume_test.go @@ -0,0 +1,26 @@ +package task_test + +import ( + "context" + "errors" + "fmt" + "github.com/flanksource/clicky/task" +) + +func ExampleRetryMessage() { + config := task.RetryConfig{OnError: func(ctx context.Context, request task.RetryRequest) (task.RetryDecision, error) { + // A terminal, approval broker, or web host can await the operator here. + return task.RetryDecision{Action: task.RetryResume, Message: "use the corrected configuration"}, nil + }} + err := task.Retry(context.Background(), config, func(ctx context.Context, attempt int) error { + if attempt == 1 { + return errors.New("invalid configuration") + } + fmt.Println(task.RetryMessage(ctx)) + return nil + }) + fmt.Println(err) + // Output: + // use the corrected configuration + // +} diff --git a/task/retry_resume.go b/task/retry_resume.go new file mode 100644 index 00000000..d4a33c2f --- /dev/null +++ b/task/retry_resume.go @@ -0,0 +1,79 @@ +package task + +import ( + "context" + "errors" + "fmt" + "time" +) + +type RetryAction string + +// RetryResume can resume after MaxRetries; RetryAutomatic keeps the bounded policy. +const ( + RetryAutomatic RetryAction = "automatic" + RetryResume RetryAction = "retry" + RetryFail RetryAction = "fail" +) + +type RetryRequest struct { + Attempt int `json:"attempt"` + Error string `json:"error"` + Automatic bool `json:"automatic"` + RemainingRetries int `json:"remainingRetries"` + Cause error `json:"-"` +} + +type RetryDecision struct { + Action RetryAction `json:"action"` + Message string `json:"message,omitempty"` +} + +type retryMessageKey struct{} + +// RetryMessage is the user's message for the resumed attempt. It is never +// logged or included in snapshots; the operation decides how to use it. +func RetryMessage(ctx context.Context) string { + message, _ := ctx.Value(retryMessageKey{}).(string) + return message +} + +func retryAfterError(ctx context.Context, config RetryConfig, attempt int, cause error) (context.Context, bool, error) { + if ctx.Err() != nil { + return ctx, false, errors.Join(cause, context.Cause(ctx)) + } + retryable := shouldRetryError(cause, config) + automatic := retryable && attempt <= config.MaxRetries + if config.OnError != nil { + decision, err := config.OnError(ctx, RetryRequest{Attempt: attempt, Error: cause.Error(), Cause: cause, Automatic: automatic, RemainingRetries: max(0, config.MaxRetries-attempt+1)}) + if err != nil { + return ctx, false, errors.Join(cause, fmt.Errorf("retry decision: %w", err)) + } + if ctx.Err() != nil { + return ctx, false, errors.Join(cause, context.Cause(ctx)) + } + switch decision.Action { + case RetryResume: + return context.WithValue(ctx, retryMessageKey{}, decision.Message), true, nil + case RetryFail: + return ctx, false, cause + case RetryAutomatic: + default: + return ctx, false, errors.Join(cause, fmt.Errorf("invalid retry decision %q", decision.Action)) + } + } + if !automatic { + if retryable && attempt > config.MaxRetries { + cause = fmt.Errorf("failed after %d attempts: %w", attempt, cause) + } + return ctx, false, cause + } + timer := time.NewTimer(config.Delay(attempt)) + defer timer.Stop() + select { + case <-timer.C: + return ctx, true, nil + case <-ctx.Done(): + return ctx, false, errors.Join(cause, context.Cause(ctx)) + } +} diff --git a/task/retry_resume_test.go b/task/retry_resume_test.go new file mode 100644 index 00000000..1301b6e4 --- /dev/null +++ b/task/retry_resume_test.go @@ -0,0 +1,82 @@ +package task_test + +import ( + "context" + "errors" + "sync/atomic" + + "github.com/flanksource/clicky/task" + commonscontext "github.com/flanksource/commons/context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Resumable retry", func() { + It("exposes the cause and passes user input to a resumed attempt after automatic retries are exhausted", func() { + cause := errors.New("the selected configuration is invalid") + message := "The configuration has been corrected; use the new settings." + var requests []task.RetryRequest + config := task.RetryConfig{MaxRetries: 0, OnError: func(_ context.Context, request task.RetryRequest) (task.RetryDecision, error) { + requests = append(requests, request) + return task.RetryDecision{Action: task.RetryResume, Message: message}, nil + }} + calls := 0 + err := task.Retry(context.Background(), config, func(ctx context.Context, attempt int) error { + calls++ + if attempt == 1 { + return cause + } + Expect(task.RetryMessage(ctx)).To(Equal(message)) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(calls).To(Equal(2)) + Expect(requests).To(HaveLen(1)) + Expect(errors.Is(requests[0].Cause, cause)).To(BeTrue()) + Expect(requests[0].Error).To(Equal(cause.Error())) + Expect(requests[0].Automatic).To(BeFalse()) + }) + It("uses the same response seam for tracked tasks", func() { + var calls atomic.Int32 + config := task.RetryConfig{OnError: func(context.Context, task.RetryRequest) (task.RetryDecision, error) { + return task.RetryDecision{Action: task.RetryResume, Message: "continue"}, nil + }} + handle := task.StartTask("Resumable operation", func(ctx commonscontext.Context, _ *task.Task) (string, error) { + if calls.Add(1) == 1 { + return "", errors.New("operator input required") + } + return task.RetryMessage(ctx), nil + }, task.WithRetryConfig(config)) + result, err := handle.GetResult() + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("continue")) + Expect(calls.Load()).To(Equal(int32(2))) + }) + It("preserves the original cause when the operator declines to retry", func() { + cause := errors.New("invalid project settings") + config := task.RetryConfig{OnError: func(context.Context, task.RetryRequest) (task.RetryDecision, error) { + return task.RetryDecision{Action: task.RetryFail}, nil + }} + Expect(task.Retry(context.Background(), config, func(context.Context, int) error { return cause })).To(MatchError(cause)) + }) + It("publishes the pending error and cancels a task waiting for an operator", func() { + waiting := make(chan struct{}) + cause := errors.New("credentials need attention") + config := task.RetryConfig{OnError: func(ctx context.Context, request task.RetryRequest) (task.RetryDecision, error) { + close(waiting) + <-ctx.Done() + return task.RetryDecision{}, context.Cause(ctx) + }} + handle := task.StartTask("Waiting operation", func(commonscontext.Context, *task.Task) (bool, error) { return false, cause }, task.WithRetryConfig(config), task.WithCancellationDrain()) + Eventually(waiting).Should(BeClosed()) + snapshot := task.SnapshotTask(handle.GetTask(), nil) + Expect(snapshot.Error).To(Equal(cause.Error())) + Expect(snapshot.Retry).NotTo(BeNil()) + Expect(snapshot.Retry.Error).To(Equal(cause.Error())) + handle.Cancel() + _, err := handle.GetResult() + Expect(err).To(HaveOccurred()) + Expect(handle.GetTask().Status()).To(Equal(task.StatusCancelled)) + Expect(task.SnapshotTask(handle.GetTask(), nil).Retry).To(BeNil()) + }) +}) diff --git a/task/snapshot.go b/task/snapshot.go index 48e596aa..b4f4007a 100644 --- a/task/snapshot.go +++ b/task/snapshot.go @@ -37,6 +37,7 @@ type TaskSnapshot struct { Status string `json:"status"` Duration string `json:"duration,omitempty"` Error string `json:"error,omitempty"` + Retry *RetryRequest `json:"retry,omitempty"` Message string `json:"message,omitempty"` // latest log line Logs []LogEntry `json:"logs,omitempty"` // all log entries Total int `json:"total,omitempty"` // group: total child tasks @@ -85,6 +86,10 @@ func SnapshotTask(t *Task, group *Group) TaskSnapshot { Status: string(t.Status()), } t.mu.Lock() + if t.retryRequest != nil && t.status == StatusRunning { + request := *t.retryRequest + snap.Retry = &request + } dependencies := append([]*Task(nil), t.dependencies...) t.mu.Unlock() for _, dependency := range dependencies { diff --git a/task/task.go b/task/task.go index 2c1c50ec..d1031a07 100644 --- a/task/task.go +++ b/task/task.go @@ -42,11 +42,12 @@ type Task struct { bufferedLogger *logger.BufferedLogger // Structs - mu sync.Mutex - doneOnce sync.Once // Ensure done channel is closed only once - drainedOnce sync.Once // Ensure drained channel is closed only once - loggerOnce sync.Once // Ensure bufferedLogger is initialized only once - retryConfig RetryConfig + mu sync.Mutex + doneOnce sync.Once // Ensure done channel is closed only once + drainedOnce sync.Once // Ensure drained channel is closed only once + loggerOnce sync.Once // Ensure bufferedLogger is initialized only once + retryConfig RetryConfig + retryRequest *RetryRequest // 8-byte aligned types startTime time.Time diff --git a/task/task_types.go b/task/task_types.go index 5f072443..21fc41f1 100644 --- a/task/task_types.go +++ b/task/task_types.go @@ -122,6 +122,9 @@ type WaitResult struct { // RetryConfig holds configuration for task retry behavior type RetryConfig struct { + // OnError exposes a failed attempt to its host. The host may wait for a user + // decision, then resume the same operation with an optional message. + OnError func(context.Context, RetryRequest) (RetryDecision, error) RetryableErrors []string // Error message patterns that should trigger retries // Retryable, when set, decides retryability on its own (e.g. errors.Is // against a sentinel); RetryableErrors is then not consulted. @@ -164,18 +167,13 @@ func Retry(ctx context.Context, config RetryConfig, run func(ctx context.Context return context.Cause(ctx) } err := run(ctx, attempt) - if err == nil || !shouldRetryError(err, config) { + if err == nil { return err } - if attempt > config.MaxRetries { - return fmt.Errorf("failed after %d attempts: %w", attempt, err) - } - timer := time.NewTimer(config.Delay(attempt)) - select { - case <-timer.C: - case <-ctx.Done(): - timer.Stop() - return fmt.Errorf("retry cancelled after attempt %d (%v): %w", attempt, err, context.Cause(ctx)) + var retry bool + ctx, retry, err = retryAfterError(ctx, config, attempt, err) + if !retry { + return err } } } diff --git a/task/worker.go b/task/worker.go index ca214271..4a596a9c 100644 --- a/task/worker.go +++ b/task/worker.go @@ -215,6 +215,17 @@ func (t *Task) armTimeout() context.CancelFunc { // executeWithRetry handles task execution with exponential backoff retry func (w *worker) executeWithRetry(task *Task) { + config := task.retryConfig + if handler := config.OnError; handler != nil { + config.OnError = func(ctx context.Context, request RetryRequest) (RetryDecision, error) { + task.mu.Lock() + task.retryRequest = &request + task.mu.Unlock() + task.dirty.Store(true) + defer func() { task.mu.Lock(); task.retryRequest = nil; task.mu.Unlock(); task.dirty.Store(true) }() + return handler(ctx, request) + } + } for { // Check if task has a function to run if task.runFunc == nil { @@ -239,32 +250,26 @@ func (w *worker) executeWithRetry(task *Task) { } if err != nil { - // Check if error is retryable - shouldRetry := shouldRetryError(err, task.retryConfig) - - if shouldRetry && task.retryCount < task.retryConfig.MaxRetries { + task.dirty.Store(true) + next, retry, retryErr := retryAfterError(task.flanksourceCtx, config, task.retryCount+1, err) + if retry { + task.mu.Lock() task.retryCount++ - // Log retry attempt to bufferedLogger + task.flanksourceCtx.Context = next + task.ctx = task.flanksourceCtx + task.mu.Unlock() task.getBufferedLogger().Warnf("Attempt %d failed, retrying: %v", task.retryCount, err) - - // Calculate backoff delay - delay := task.retryConfig.Delay(task.retryCount) - - // Wait for delay or cancellation - select { - case <-time.After(delay): - continue // Retry - case <-task.ctx.Done(): - task.Cancel() - return - } - } else { - if _, failErr := task.FailedWithError(err); failErr != nil { - // Log error but continue - task is already in failed state - task.getBufferedLogger().Errorf("Failed to set error status: %v", failErr) - } + continue + } + if task.ctx.Err() != nil { + task.Cancel() return } + task.mu.Lock() + task.err = retryErr + task.mu.Unlock() + _, _ = task.FailedWithError(retryErr) + return } else { task.Success() return