From 1bec767ef458e390f14d045c35e45f7b7d858575 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 2 Nov 2026 07:18:31 +0200 Subject: [PATCH] feat(task): add weighted task slots and zero-slot supervisor dispatch Allow supervisors to await child tasks without consuming ordinary worker slots while still respecting global and group capacity. Expose WithSlots for weighted reservations, dispatch zero-slot tasks through supervisors, and include active supervisors in foreground waits. --- aliases.go | 1 + task/group_admission.go | 6 +- task/manager.go | 7 +++ task/manager_wait.go | 6 ++ task/slots.go | 61 ++++++++++++++++++++ task/slots_ginkgo_test.go | 99 ++++++++++++++++++++++++++++++++ task/task.go | 1 + task/task_wait.go | 2 +- task/worker.go | 116 +++++++++++++++++++++++--------------- 9 files changed, 250 insertions(+), 49 deletions(-) create mode 100644 task/slots.go create mode 100644 task/slots_ginkgo_test.go diff --git a/aliases.go b/aliases.go index 70c1af84..5740e698 100644 --- a/aliases.go +++ b/aliases.go @@ -43,6 +43,7 @@ var ( DefaultRetryConfig = task.DefaultRetryConfig DefaultTaskManagerOptions = task.DefaultManagerOptions WithTimeout = task.WithTimeout + WithSlots = task.WithSlots WithTaskTimeout = task.WithTaskTimeout WithDependencies = task.WithDependencies WithFunc = task.WithFunc diff --git a/task/group_admission.go b/task/group_admission.go index 22068533..66db468d 100644 --- a/task/group_admission.go +++ b/task/group_admission.go @@ -97,7 +97,7 @@ func (g *Group) admitLocked(t *Task) bool { }() for i, member := range l.waiting { if member == t { - admitted := g.sem.TryAcquire(1) + admitted := g.sem.TryAcquire(int64(t.slots)) if admitted { kept = append(kept, l.waiting[i+1:]...) } else { @@ -120,10 +120,10 @@ func (g *Group) admitLocked(t *Task) bool { } // release returns a member's permit and wakes the line. -func (g *Group) release() { +func (g *Group) release(slots int) { g.admission.mu.Lock() defer g.admission.mu.Unlock() - g.sem.Release(1) + g.sem.Release(int64(slots)) g.wakeLocked() } diff --git a/task/manager.go b/task/manager.go index d95e4dba..5af0b8eb 100644 --- a/task/manager.go +++ b/task/manager.go @@ -29,6 +29,8 @@ type Manager struct { width int verbose atomic.Bool maxConcurrent int + slotsUsed int + supervisors sync.Map // *Task -> *worker outside the ordinary worker pool // workerLimit is how many workers may take tasks (SetMaxConcurrent): a // worker whose id is at or above it idles. Zero lets every worker run. workerLimit atomic.Int32 @@ -410,6 +412,7 @@ func (tm *Manager) newTask(name string, opts ...Option) *Task { flanksourceCtx: flanksourceCtx, retryConfig: tm.retryConfig, retryCount: 0, + slots: 1, doneChan: make(chan struct{}), drainedChan: make(chan struct{}), } @@ -466,6 +469,10 @@ func (tm *Manager) enqueueClaimed(task *Task) { if task.groupSem() != nil { task.parent.admission.enlist(task) } + if task.slots == 0 { + go tm.runSupervisor(task) + return + } tm.taskQueue.Enqueue(task) } diff --git a/task/manager_wait.go b/task/manager_wait.go index 4cb0b2dd..1aba4012 100644 --- a/task/manager_wait.go +++ b/task/manager_wait.go @@ -151,6 +151,12 @@ func (tm *Manager) foregroundWorkersActive() int { active++ } } + tm.supervisors.Range(func(_, value any) bool { + if t := value.(*worker).inflight.Load(); t != nil && !t.IsBackground() { + active++ + } + return true + }) return active } diff --git a/task/slots.go b/task/slots.go new file mode 100644 index 00000000..018aa88d --- /dev/null +++ b/task/slots.go @@ -0,0 +1,61 @@ +package task + +import ( + "fmt" + "time" +) + +// WithSlots sets the concurrency capacity reserved by a task. Ordinary tasks +// use one slot. Supervisors use zero while waiting for their child tasks. +func WithSlots(slots int) Option { + if slots < 0 { + panic("task: slots must be non-negative") + } + return func(t *Task) { t.slots = slots } +} + +func (tm *Manager) reserveSlots(t *Task) (bool, error) { + tm.mu.Lock() + defer tm.mu.Unlock() + if tm.maxConcurrent > 0 && t.slots > tm.maxConcurrent { + return false, fmt.Errorf("task %q requests %d slots, global capacity is %d", t.name, t.slots, tm.maxConcurrent) + } + if t.parent != nil && t.parent.concurrency > 0 && t.slots > t.parent.concurrency { + return false, fmt.Errorf("task %q requests %d slots, group capacity is %d", t.name, t.slots, t.parent.concurrency) + } + if tm.maxConcurrent > 0 && tm.slotsUsed+t.slots > tm.maxConcurrent { + return false, nil + } + tm.slotsUsed += t.slots + return true, nil +} + +func (tm *Manager) releaseSlots(t *Task) { + tm.mu.Lock() + tm.slotsUsed -= t.slots + tm.mu.Unlock() +} + +// Dispatch supervisors independently because all ordinary workers may already +// be occupied by children waiting for another supervisor. +func (tm *Manager) runSupervisor(t *Task) { + w := &worker{manager: tm} + w.inflight.Store(t) + tm.supervisors.Store(t, w) + defer tm.supervisors.Delete(t) + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for !retiredBeforeStart(t) && !w.checkDependencies(t) { + select { + case <-t.ctx.Done(): + t.Cancel() + t.markCompleted() + t.signalDone() + tm.releaseIdentity(t) + observeGroupTerminal(t.parent) + return + case <-ticker.C: + } + } + w.finishTask(t, false) +} diff --git a/task/slots_ginkgo_test.go b/task/slots_ginkgo_test.go new file mode 100644 index 00000000..455495e3 --- /dev/null +++ b/task/slots_ginkgo_test.go @@ -0,0 +1,99 @@ +package task + +import ( + "context" + "errors" + "time" + + fc "github.com/flanksource/commons/context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("task slots", func() { + var original *Manager + BeforeEach(func() { original = global; global = newTestManager(1) }) + AfterEach(func() { global.awaitAll(); global.stopRender(); global = original }) + + It("allows nested supervisors to await children with one global and group slot", func() { + group := StartGroup[int]("supervisors", WithConcurrency(1)) + outer := group.Add("outer", func(ctx fc.Context, _ *Task) (int, error) { + inner := group.Add("inner", func(ctx fc.Context, _ *Task) (int, error) { + return group.Add("child", func(fc.Context, *Task) (int, error) { return 7, nil }, WithContext(ctx)).GetResult() + }, WithSlots(0), WithContext(ctx)) + return inner.GetResult() + }, WithSlots(0), WithTimeout(time.Second), WithCancellationDrain()) + value, err := outer.GetResult() + Expect(err).NotTo(HaveOccurred()) + Expect(value).To(Equal(7)) + }) + + It("starts a supervisor while ordinary capacity is occupied and includes it in waits", func() { + gate := make(chan struct{}) + defer close(gate) + started := make(chan struct{}) + StartTask("occupied", func(fc.Context, *Task) (int, error) { close(started); <-gate; return 0, nil }) + Eventually(started).Should(BeClosed()) + supervisor := StartTask("supervisor", func(fc.Context, *Task) (int, error) { return 9, nil }, WithSlots(0)) + Eventually(supervisor.Task.IsOk).Should(BeTrue()) + Expect(global.allTasksCompleted()).To(BeFalse()) + }) + + It("rejects invalid reservations", func() { + Expect(func() { WithSlots(-1) }).To(Panic()) + _, err := StartTask("impossible", func(fc.Context, *Task) (int, error) { return 0, nil }, WithSlots(2)).GetResult() + Expect(err).To(MatchError(ContainSubstring("slots"))) + }) + + It("keeps global waits blocked on a foreground supervisor", func() { + gate := make(chan struct{}) + started := make(chan struct{}) + run := StartTask("waiting supervisor", func(fc.Context, *Task) (int, error) { close(started); <-gate; return 0, nil }, WithSlots(0)) + Eventually(started).Should(BeClosed()) + done := make(chan struct{}) + go func() { global.awaitAll(); close(done) }() + Consistently(done, 50*time.Millisecond).ShouldNot(BeClosed()) + close(gate) + _, err := run.GetResult() + Expect(err).NotTo(HaveOccurred()) + Eventually(done).Should(BeClosed()) + }) + + It("reserves weighted global and group capacity until callbacks return", func() { + SetMaxConcurrent(3) + group := StartGroup[int]("weighted", WithConcurrency(2)) + gate, started, next := make(chan struct{}), make(chan struct{}), make(chan struct{}) + first := group.Add("two slots", func(fc.Context, *Task) (int, error) { close(started); <-gate; return 0, nil }, WithSlots(2)) + Eventually(started).Should(BeClosed()) + second := group.Add("one slot", func(fc.Context, *Task) (int, error) { close(next); return 0, nil }) + Consistently(next, 50*time.Millisecond).ShouldNot(BeClosed()) + close(gate) + _, err := first.GetResult() + Expect(err).NotTo(HaveOccurred()) + _, err = second.GetResult() + Expect(err).NotTo(HaveOccurred()) + }) + + It("propagates dependency failure to a zero-slot supervisor", func() { + failed := StartTask("failure", func(fc.Context, *Task) (int, error) { return 0, errors.New("broken") }, WithRetryConfig(RetryConfig{MaxRetries: 0})) + run := StartTask("dependent", func(fc.Context, *Task) (int, error) { return 1, nil }, WithSlots(0), WithDependencies(failed.Task)) + _, err := run.GetResult() + Expect(err).To(MatchError("dependency failed")) + }) + + It("drains a cancelled supervisor callback before returning", func() { + ctx, cancel := context.WithCancel(context.Background()) + started, returned := make(chan struct{}), make(chan struct{}) + run := StartTask("cancel", func(ctx fc.Context, _ *Task) (int, error) { + close(started) + <-ctx.Done() + close(returned) + return 0, ctx.Err() + }, WithSlots(0), WithContext(ctx), WithCancellationDrain()) + Eventually(started).Should(BeClosed()) + cancel() + _, err := run.GetResult() + Expect(errors.Is(err, context.Canceled)).To(BeTrue()) + Expect(returned).To(BeClosed()) + }) +}) diff --git a/task/task.go b/task/task.go index 9b6527f5..cf25c143 100644 --- a/task/task.go +++ b/task/task.go @@ -71,6 +71,7 @@ type Task struct { maxValue int retryCount int priority int // Priority for queue ordering (lower = higher priority) + slots int // Immutable reservation; zero denotes a supervisor plainLogsRendered int // buffered log entries already emitted by PlainRender; guarded by mu // Smaller types diff --git a/task/task_wait.go b/task/task_wait.go index af57aa5f..6e7c6501 100644 --- a/task/task_wait.go +++ b/task/task_wait.go @@ -166,7 +166,7 @@ func (t *Task) IsGroup() bool { // assigned once in StartGroup before any task is added and never mutated, so no // lock is needed. func (t *Task) groupSem() *semaphore.Weighted { - if t.parent == nil { + if t.parent == nil || t.slots == 0 { return nil } return t.parent.sem diff --git a/task/worker.go b/task/worker.go index 39a77975..6c2400c0 100644 --- a/task/worker.go +++ b/task/worker.go @@ -74,58 +74,84 @@ func (w *worker) run() { // pool is saturated, or a member added before this one is still // waiting for a permit, do not occupy a worker slot: admit parks the // task in its group's line, which re-enqueues it once it can start. + reserved, err := w.manager.reserveSlots(task) + if err != nil { + task.mu.Lock() + task.err = err + task.mu.Unlock() + task.Failed() + task.markCompleted() + task.signalDone() + w.manager.releaseIdentity(task) + observeGroupTerminal(task.parent) + if task.groupSem() != nil { + task.parent.wake() + } + continue + } + if !reserved { + w.manager.taskQueue.EnqueueWithDelay(task, 10*time.Millisecond) + continue + } sem := task.groupSem() if sem != nil && !task.parent.admit(task) { + w.manager.releaseSlots(task) continue } + w.finishTask(task, true) + } + } +} - w.inflight.Store(task) - w.manager.workersActive.Add(1) +func (w *worker) finishTask(task *Task, reserved bool) { + w.inflight.Store(task) + w.manager.workersActive.Add(1) - func() { - // Release the group permit on ALL exit paths (success, failure, - // retry, timeout, panic). Registered first so it runs last, - // after the recover defer records terminal status. - if sem != nil { - defer task.parent.release() - } - // Flip completed before the permit is released: the permit is - // what admits the next task, and a dependent that dequeues while - // this one is terminal but not yet completed is cancelled as - // "dependency failed". - defer task.markCompleted() - defer func() { - if r := recover(); r != nil { - task.mu.Lock() - task.status = StatusFailed - task.err = fmt.Errorf("panic: %v", r) - task.endTime = time.Now() - task.mu.Unlock() - } - }() - w.executeTask(task) - }() - - w.manager.workersActive.Add(-1) - - w.manager.releaseIdentity(task) - - // Signal done channel for compatibility - task.signalDone() - - // Finishing the last task in a group is what makes the group - // terminal, and this is the only place that fact is known without - // something happening to snapshot it. Observing it here is what - // gives a run a durable record the moment it ends rather than - // whenever a UI next looks. - observeGroupTerminal(task.parent) - - // Cleared only after all post-run bookkeeping so a wait that gates - // on foregroundWorkersActive never returns before identity cleanup - // and the done signal have landed. - w.inflight.Store(nil) + func() { + // Release the group permit on ALL exit paths (success, failure, + // retry, timeout, panic). Registered first so it runs last, + // after the recover defer records terminal status. + if reserved { + defer w.manager.releaseSlots(task) } - } + if task.groupSem() != nil { + defer task.parent.release(task.slots) + } + // Flip completed before the permit is released: the permit is + // what admits the next task, and a dependent that dequeues while + // this one is terminal but not yet completed is cancelled as + // "dependency failed". + defer task.markCompleted() + defer func() { + if r := recover(); r != nil { + task.mu.Lock() + task.status = StatusFailed + task.err = fmt.Errorf("panic: %v", r) + task.endTime = time.Now() + task.mu.Unlock() + } + }() + w.executeTask(task) + }() + + w.manager.workersActive.Add(-1) + + w.manager.releaseIdentity(task) + + // Signal done channel for compatibility + task.signalDone() + + // Finishing the last task in a group is what makes the group + // terminal, and this is the only place that fact is known without + // something happening to snapshot it. Observing it here is what + // gives a run a durable record the moment it ends rather than + // whenever a UI next looks. + observeGroupTerminal(task.parent) + + // Cleared only after all post-run bookkeeping so a wait that gates + // on foregroundWorkersActive never returns before identity cleanup + // and the done signal have landed. + w.inflight.Store(nil) } // checkDependencies reports whether a task may start: every dependency has to