Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions aliases.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ var (
DefaultRetryConfig = task.DefaultRetryConfig
DefaultTaskManagerOptions = task.DefaultManagerOptions
WithTimeout = task.WithTimeout
WithSlots = task.WithSlots
WithTaskTimeout = task.WithTaskTimeout
WithDependencies = task.WithDependencies
WithFunc = task.WithFunc
Expand Down
6 changes: 3 additions & 3 deletions task/group_admission.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func (g *Group) admitLocked(t *Task) bool {
}()
for i, member := range l.waiting {
if member == t {
admitted := g.sem.TryAcquire(1)
admitted := g.sem.TryAcquire(int64(t.slots))
if admitted {
kept = append(kept, l.waiting[i+1:]...)
} else {
Expand All @@ -120,10 +120,10 @@ func (g *Group) admitLocked(t *Task) bool {
}

// release returns a member's permit and wakes the line.
func (g *Group) release() {
func (g *Group) release(slots int) {
g.admission.mu.Lock()
defer g.admission.mu.Unlock()
g.sem.Release(1)
g.sem.Release(int64(slots))
g.wakeLocked()
}

Expand Down
7 changes: 7 additions & 0 deletions task/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ type Manager struct {
width int
verbose atomic.Bool
maxConcurrent int
slotsUsed int
supervisors sync.Map // *Task -> *worker outside the ordinary worker pool
// workerLimit is how many workers may take tasks (SetMaxConcurrent): a
// worker whose id is at or above it idles. Zero lets every worker run.
workerLimit atomic.Int32
Expand Down Expand Up @@ -410,6 +412,7 @@ func (tm *Manager) newTask(name string, opts ...Option) *Task {
flanksourceCtx: flanksourceCtx,
retryConfig: tm.retryConfig,
retryCount: 0,
slots: 1,
doneChan: make(chan struct{}),
drainedChan: make(chan struct{}),
}
Expand Down Expand Up @@ -466,6 +469,10 @@ func (tm *Manager) enqueueClaimed(task *Task) {
if task.groupSem() != nil {
task.parent.admission.enlist(task)
}
if task.slots == 0 {
go tm.runSupervisor(task)
return
}
tm.taskQueue.Enqueue(task)
}

Expand Down
6 changes: 6 additions & 0 deletions task/manager_wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,12 @@ func (tm *Manager) foregroundWorkersActive() int {
active++
}
}
tm.supervisors.Range(func(_, value any) bool {
if t := value.(*worker).inflight.Load(); t != nil && !t.IsBackground() {
active++
}
return true
})
return active
}

Expand Down
61 changes: 61 additions & 0 deletions task/slots.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package task

import (
"fmt"
"time"
)

// WithSlots sets the concurrency capacity reserved by a task. Ordinary tasks
// use one slot. Supervisors use zero while waiting for their child tasks.
func WithSlots(slots int) Option {
if slots < 0 {
panic("task: slots must be non-negative")
}
return func(t *Task) { t.slots = slots }
}

func (tm *Manager) reserveSlots(t *Task) (bool, error) {
tm.mu.Lock()
defer tm.mu.Unlock()
if tm.maxConcurrent > 0 && t.slots > tm.maxConcurrent {
return false, fmt.Errorf("task %q requests %d slots, global capacity is %d", t.name, t.slots, tm.maxConcurrent)
}
if t.parent != nil && t.parent.concurrency > 0 && t.slots > t.parent.concurrency {
return false, fmt.Errorf("task %q requests %d slots, group capacity is %d", t.name, t.slots, t.parent.concurrency)
}
if tm.maxConcurrent > 0 && tm.slotsUsed+t.slots > tm.maxConcurrent {
return false, nil
}
tm.slotsUsed += t.slots
return true, nil
}

func (tm *Manager) releaseSlots(t *Task) {
tm.mu.Lock()
tm.slotsUsed -= t.slots
tm.mu.Unlock()
}

// Dispatch supervisors independently because all ordinary workers may already
// be occupied by children waiting for another supervisor.
func (tm *Manager) runSupervisor(t *Task) {
w := &worker{manager: tm}
w.inflight.Store(t)
tm.supervisors.Store(t, w)
defer tm.supervisors.Delete(t)
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for !retiredBeforeStart(t) && !w.checkDependencies(t) {
select {
case <-t.ctx.Done():
t.Cancel()
t.markCompleted()
t.signalDone()
tm.releaseIdentity(t)
observeGroupTerminal(t.parent)
return
case <-ticker.C:
}
}
w.finishTask(t, false)
}
99 changes: 99 additions & 0 deletions task/slots_ginkgo_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package task

import (
"context"
"errors"
"time"

fc "github.com/flanksource/commons/context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("task slots", func() {
var original *Manager
BeforeEach(func() { original = global; global = newTestManager(1) })
AfterEach(func() { global.awaitAll(); global.stopRender(); global = original })

It("allows nested supervisors to await children with one global and group slot", func() {
group := StartGroup[int]("supervisors", WithConcurrency(1))
outer := group.Add("outer", func(ctx fc.Context, _ *Task) (int, error) {
inner := group.Add("inner", func(ctx fc.Context, _ *Task) (int, error) {
return group.Add("child", func(fc.Context, *Task) (int, error) { return 7, nil }, WithContext(ctx)).GetResult()
}, WithSlots(0), WithContext(ctx))
return inner.GetResult()
}, WithSlots(0), WithTimeout(time.Second), WithCancellationDrain())
value, err := outer.GetResult()
Expect(err).NotTo(HaveOccurred())
Expect(value).To(Equal(7))
})

It("starts a supervisor while ordinary capacity is occupied and includes it in waits", func() {
gate := make(chan struct{})
defer close(gate)
started := make(chan struct{})
StartTask("occupied", func(fc.Context, *Task) (int, error) { close(started); <-gate; return 0, nil })
Eventually(started).Should(BeClosed())
supervisor := StartTask("supervisor", func(fc.Context, *Task) (int, error) { return 9, nil }, WithSlots(0))
Eventually(supervisor.Task.IsOk).Should(BeTrue())
Expect(global.allTasksCompleted()).To(BeFalse())
})

It("rejects invalid reservations", func() {
Expect(func() { WithSlots(-1) }).To(Panic())
_, err := StartTask("impossible", func(fc.Context, *Task) (int, error) { return 0, nil }, WithSlots(2)).GetResult()
Expect(err).To(MatchError(ContainSubstring("slots")))
})

It("keeps global waits blocked on a foreground supervisor", func() {
gate := make(chan struct{})
started := make(chan struct{})
run := StartTask("waiting supervisor", func(fc.Context, *Task) (int, error) { close(started); <-gate; return 0, nil }, WithSlots(0))
Eventually(started).Should(BeClosed())
done := make(chan struct{})
go func() { global.awaitAll(); close(done) }()
Consistently(done, 50*time.Millisecond).ShouldNot(BeClosed())
close(gate)
_, err := run.GetResult()
Expect(err).NotTo(HaveOccurred())
Eventually(done).Should(BeClosed())
})

It("reserves weighted global and group capacity until callbacks return", func() {
SetMaxConcurrent(3)
group := StartGroup[int]("weighted", WithConcurrency(2))
gate, started, next := make(chan struct{}), make(chan struct{}), make(chan struct{})
first := group.Add("two slots", func(fc.Context, *Task) (int, error) { close(started); <-gate; return 0, nil }, WithSlots(2))
Eventually(started).Should(BeClosed())
second := group.Add("one slot", func(fc.Context, *Task) (int, error) { close(next); return 0, nil })
Consistently(next, 50*time.Millisecond).ShouldNot(BeClosed())
close(gate)
_, err := first.GetResult()
Expect(err).NotTo(HaveOccurred())
_, err = second.GetResult()
Expect(err).NotTo(HaveOccurred())
})

It("propagates dependency failure to a zero-slot supervisor", func() {
failed := StartTask("failure", func(fc.Context, *Task) (int, error) { return 0, errors.New("broken") }, WithRetryConfig(RetryConfig{MaxRetries: 0}))
run := StartTask("dependent", func(fc.Context, *Task) (int, error) { return 1, nil }, WithSlots(0), WithDependencies(failed.Task))
_, err := run.GetResult()
Expect(err).To(MatchError("dependency failed"))
})

It("drains a cancelled supervisor callback before returning", func() {
ctx, cancel := context.WithCancel(context.Background())
started, returned := make(chan struct{}), make(chan struct{})
run := StartTask("cancel", func(ctx fc.Context, _ *Task) (int, error) {
close(started)
<-ctx.Done()
close(returned)
return 0, ctx.Err()
}, WithSlots(0), WithContext(ctx), WithCancellationDrain())
Eventually(started).Should(BeClosed())
cancel()
_, err := run.GetResult()
Expect(errors.Is(err, context.Canceled)).To(BeTrue())
Expect(returned).To(BeClosed())
})
})
1 change: 1 addition & 0 deletions task/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ type Task struct {
maxValue int
retryCount int
priority int // Priority for queue ordering (lower = higher priority)
slots int // Immutable reservation; zero denotes a supervisor
plainLogsRendered int // buffered log entries already emitted by PlainRender; guarded by mu

// Smaller types
Expand Down
2 changes: 1 addition & 1 deletion task/task_wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func (t *Task) IsGroup() bool {
// assigned once in StartGroup before any task is added and never mutated, so no
// lock is needed.
func (t *Task) groupSem() *semaphore.Weighted {
if t.parent == nil {
if t.parent == nil || t.slots == 0 {
return nil
}
return t.parent.sem
Expand Down
116 changes: 71 additions & 45 deletions task/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,58 +74,84 @@ func (w *worker) run() {
// pool is saturated, or a member added before this one is still
// waiting for a permit, do not occupy a worker slot: admit parks the
// task in its group's line, which re-enqueues it once it can start.
reserved, err := w.manager.reserveSlots(task)
if err != nil {
task.mu.Lock()
task.err = err
task.mu.Unlock()
task.Failed()
task.markCompleted()
task.signalDone()
w.manager.releaseIdentity(task)
observeGroupTerminal(task.parent)
if task.groupSem() != nil {
task.parent.wake()
}
continue
}
if !reserved {
w.manager.taskQueue.EnqueueWithDelay(task, 10*time.Millisecond)
continue
}
sem := task.groupSem()
if sem != nil && !task.parent.admit(task) {
w.manager.releaseSlots(task)
continue
}
w.finishTask(task, true)
}
}
}

w.inflight.Store(task)
w.manager.workersActive.Add(1)
func (w *worker) finishTask(task *Task, reserved bool) {
w.inflight.Store(task)
w.manager.workersActive.Add(1)

func() {
// Release the group permit on ALL exit paths (success, failure,
// retry, timeout, panic). Registered first so it runs last,
// after the recover defer records terminal status.
if sem != nil {
defer task.parent.release()
}
// Flip completed before the permit is released: the permit is
// what admits the next task, and a dependent that dequeues while
// this one is terminal but not yet completed is cancelled as
// "dependency failed".
defer task.markCompleted()
defer func() {
if r := recover(); r != nil {
task.mu.Lock()
task.status = StatusFailed
task.err = fmt.Errorf("panic: %v", r)
task.endTime = time.Now()
task.mu.Unlock()
}
}()
w.executeTask(task)
}()

w.manager.workersActive.Add(-1)

w.manager.releaseIdentity(task)

// Signal done channel for compatibility
task.signalDone()

// Finishing the last task in a group is what makes the group
// terminal, and this is the only place that fact is known without
// something happening to snapshot it. Observing it here is what
// gives a run a durable record the moment it ends rather than
// whenever a UI next looks.
observeGroupTerminal(task.parent)

// Cleared only after all post-run bookkeeping so a wait that gates
// on foregroundWorkersActive never returns before identity cleanup
// and the done signal have landed.
w.inflight.Store(nil)
func() {
// Release the group permit on ALL exit paths (success, failure,
// retry, timeout, panic). Registered first so it runs last,
// after the recover defer records terminal status.
if reserved {
defer w.manager.releaseSlots(task)
}
}
if task.groupSem() != nil {
defer task.parent.release(task.slots)
}
// Flip completed before the permit is released: the permit is
// what admits the next task, and a dependent that dequeues while
// this one is terminal but not yet completed is cancelled as
// "dependency failed".
defer task.markCompleted()
defer func() {
if r := recover(); r != nil {
task.mu.Lock()
task.status = StatusFailed
task.err = fmt.Errorf("panic: %v", r)
task.endTime = time.Now()
task.mu.Unlock()
}
}()
w.executeTask(task)
}()

w.manager.workersActive.Add(-1)

w.manager.releaseIdentity(task)

// Signal done channel for compatibility
task.signalDone()

// Finishing the last task in a group is what makes the group
// terminal, and this is the only place that fact is known without
// something happening to snapshot it. Observing it here is what
// gives a run a durable record the moment it ends rather than
// whenever a UI next looks.
observeGroupTerminal(task.parent)

// Cleared only after all post-run bookkeeping so a wait that gates
// on foregroundWorkersActive never returns before identity cleanup
// and the done signal have landed.
w.inflight.Store(nil)
}

// checkDependencies reports whether a task may start: every dependency has to
Expand Down
Loading