-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathruntime.go
More file actions
1476 lines (1374 loc) · 55.1 KB
/
Copy pathruntime.go
File metadata and controls
1476 lines (1374 loc) · 55.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package chat
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
)
type ConcurrencyStrategy int
const (
// ConcurrencyDrop acknowledges and drops a Lock Conflict. This is the default.
ConcurrencyDrop ConcurrencyStrategy = iota
// ConcurrencyQueue waits for the in-flight handler, then dispatches the most
// recent superseded event for the scope.
ConcurrencyQueue
// ConcurrencyDebounce coalesces rapid follow-ups: each new routed event for a
// scope supersedes the previous waiter and only the final event in a
// DebounceInterval quiet period dispatches. Superseded events are surfaced as
// skipped through Runtime Observation, never silently. Requires deferred
// dispatch (a synchronous webhook cannot park an event past the platform's
// acknowledgement deadline).
//
// "Final" follows the dispatch admission order on this instance: a delivery
// delayed in its prelude never displaces a waiter admitted after it. The
// quiet period is measured over registered waiters: a delivery stalled in
// its prelude (validation, dedupe, routing) for longer than the interval
// does not reset the running timer and dispatches separately afterwards —
// still serialized by the Thread Lock and never lost, but not coalesced.
//
// Like queue supersession, coalescing is per runtime instance: events for
// one scope delivered to different instances sharing a State are not
// superseded across instances (each instance dispatches its own final
// event, serialized by the Thread Lock). Per-instance supersession is the
// decided v0.x contract (ADR 0015); cross-instance coalescing is rejected
// for now behind that ADR's reopening bar.
ConcurrencyDebounce
// ConcurrencyConcurrent is the explicit opt-out of per-scope serialization:
// every routed event dispatches immediately in its own execution, bounded by
// MaxConcurrent. No Thread Lock is taken, so the caller accepts interleaved
// replies and races on Thread Application State.
ConcurrencyConcurrent
// ConcurrencyBurst collects routed events for a scope into a batch while a
// BurstWindow collection window is open, then dispatches the batch under a
// single Thread Lock hold, running every member in join order, each with
// its own DetachTimeout execution budget. The window is anchored at its
// first member's join and is never extended by later arrivals; a window
// reaching MaxBurstBatch seals immediately (the cap-reaching member is the
// batch's last member) and the next event opens a rolled window dispatched
// strictly after its predecessor. Batch shaping is delivery-preserving
// (ADR 0015): boundaries move, but an accepted member is never dropped.
// Requires deferred dispatch (a synchronous webhook cannot park an event
// past the platform's acknowledgement deadline).
//
// Parked members hold Admission Bound slots until their terminal
// disposition, so burst retention stays inside MaxDetached. A member whose
// handler returns an error does not abort its batch; losing the Lock
// Lease mid-batch cancels the running member (ErrPreempted) and skips the
// remaining members observably rather than running them unserialized.
// Like every deferred handler, member cancellation is cooperative: a
// handler that ignores its context blocks the members behind it.
//
// Like queue supersession and debounce coalescing, batching is per runtime
// instance (ADR 0015): events for one scope delivered to different
// instances sharing a State batch independently, serialized by the Thread
// Lock. Cross-instance coalescing is rejected for now behind that ADR's
// reopening bar.
ConcurrencyBurst
)
// The force/steerability names from ADR 0012 remain reserved: per ADR 0015
// they are rejected pending that ADR's formal-design bar.
// LockScope selects what key the Thread Lock guards. The opaque Thread ID is
// unchanged; the scope only chooses the serialization key.
type LockScope int
const (
// LockScopeThread serializes handlers per Thread. This is the default.
LockScopeThread LockScope = iota
// LockScopeChannel widens serialization from a single Thread to its whole
// channel, for platforms whose model requires channel-wide ordering. A
// Thread whose adapter reports no channel falls back to per-Thread locking
// rather than sharing one adapter-wide key.
LockScopeChannel
)
// DispatchMode selects whether the routed handler runs before or after the
// adapter acknowledges the platform.
type DispatchMode int
const (
// DispatchSync runs the handler under the request context and acknowledges
// after it returns. This is the default.
DispatchSync DispatchMode = iota
// DispatchDeferred acknowledges after the prelude and runs the handler on the
// detached work context (ack-then-work).
DispatchDeferred
)
type RuntimeOptions struct {
DedupeTTL time.Duration
ThreadLockTTL time.Duration
Concurrency ConcurrencyStrategy
Dispatch DispatchMode
DetachTimeout time.Duration
// LockScope selects the Thread Lock key: per Thread (default) or per
// channel.
LockScope LockScope
// DebounceInterval is the ConcurrencyDebounce quiet period. It must be
// positive under that strategy and is ignored otherwise.
DebounceInterval time.Duration
// MaxConcurrent bounds simultaneous handler executions under
// ConcurrencyConcurrent. It must be positive under that strategy and is
// ignored otherwise.
MaxConcurrent int
// BurstWindow is the ConcurrencyBurst collection window: routed events for
// a scope collect for this long — anchored at the window's first member,
// never extended by later arrivals — before dispatching as one batch. It
// must be positive under that strategy and is ignored otherwise. The
// window is collection time, not execution time: it does not consume the
// batch's lock-wait budget or any member's DetachTimeout.
BurstWindow time.Duration
// MaxBurstBatch caps how many members one burst batch may collect. A
// window reaching the cap seals immediately, with the cap-reaching member
// as the sealed batch's last member; the next event opens a rolled window
// dispatched strictly after its predecessor. The cap shapes batches — it
// never rejects or drops an accepted member (delivery-preserving shaping,
// ADR 0015) — and parked members remain bounded by MaxDetached regardless.
// Zero disables the cap; it must not be negative under the burst strategy
// and is ignored otherwise.
MaxBurstBatch int
// MaxDetached is the deferred-dispatch Admission Bound (ADR 0015): a
// per-instance cap on admitted-but-incomplete deferred deliveries.
// Everything a delivery retains under DispatchDeferred counts against it —
// running detached tails, parked queue/debounce waiters, concurrent
// slot-waiters, and parked burst batch members — and capacity frees only
// when that retention ends. A delivery arriving at the cap is rejected
// with ErrAdmissionRejected
// before acknowledgement and before dedupe marking, so a platform retry is
// never deduped away. It must be positive under DispatchDeferred and is
// ignored under DispatchSync; DefaultRuntimeOptions sets 1024.
//
// Sizing: the bound is a count, not bytes — the runtime cannot measure
// retained platform payloads or handler closures. Budget roughly
// MaxDetached x (platform payload ceiling + one goroutine stack + whatever
// the handler closure pins) against the instance's memory; sustained
// rejection is the platform-visible backpressure signal, so size the cap
// to be reached only under genuine overload and keep front-door rate
// limiting for fleet-level control.
MaxDetached int
// MaxDetachedPerTenant additionally caps any single installation's share
// of MaxDetached, keyed on the delivery's (adapter, tenant) installation
// identity (ADR 0006). It is a ceiling through the same rejection path,
// not a reservation: a hot tenant is capped, but no capacity is guaranteed
// to the others. Zero disables the ceiling; it must not be negative under
// DispatchDeferred and is ignored under DispatchSync.
MaxDetachedPerTenant int
}
func DefaultRuntimeOptions() RuntimeOptions {
return RuntimeOptions{
DedupeTTL: 24 * time.Hour,
ThreadLockTTL: 2 * time.Minute,
Concurrency: ConcurrencyDrop,
Dispatch: DispatchSync,
DetachTimeout: 0,
LockScope: LockScopeThread,
MaxDetached: 1024,
}
}
type Option func(*config)
type config struct {
state State
adapters []Adapter
logger *slog.Logger
observer Observer
options RuntimeOptions
}
func WithState(state State) Option {
return func(cfg *config) {
cfg.state = state
}
}
func WithAdapter(adapter Adapter) Option {
return func(cfg *config) {
cfg.adapters = append(cfg.adapters, adapter)
}
}
func WithLogger(logger *slog.Logger) Option {
return func(cfg *config) {
cfg.logger = logger
}
}
func WithRuntimeOptions(options RuntimeOptions) Option {
return func(cfg *config) {
cfg.options = options
}
}
type Chat struct {
state State
adapters map[string]Adapter
logger *slog.Logger
observer Observer
options RuntimeOptions
handlersMu sync.RWMutex
newMention MessageHandler
subscribedMessage MessageHandler
command CommandHandler
interaction InteractionHandler
acceptancesMu sync.Mutex
eventAcceptances map[string]*eventAcceptance
// baseCtx is the long-lived base for the detached work context: not a request
// context, not context.Background(). Cancelled by Shutdown.
baseCtx context.Context
baseCancel context.CancelFunc
// inflight tracks detached tails so Shutdown drains them before state shutdown.
inflight sync.WaitGroup
// queueMu guards pending, the per-scope most-recent pending waiter.
queueMu sync.Mutex
pending map[string]*pendingWaiter
// dispatchSeq orders deliveries by dispatch admission so a delivery delayed
// in its prelude can never displace a newer pending waiter.
dispatchSeq atomic.Uint64
// concurrencySlots bounds simultaneous handler executions under
// ConcurrencyConcurrent; nil under every other strategy.
concurrencySlots chan struct{}
// burstMu guards burstScopes, the per-scope burst collection state.
burstMu sync.Mutex
burstScopes map[string]*burstScope
// admission is the deferred-dispatch Admission Bound (ADR 0015); nil under
// DispatchSync.
admission *admissionGate
shutdownMu sync.Mutex
shutdown bool
shutdownDone chan struct{}
}
type eventAcceptance struct {
done chan struct{}
err error
}
func New(ctx context.Context, opts ...Option) (*Chat, error) {
cfg := config{
logger: slog.Default(),
observer: noopObserver{},
options: DefaultRuntimeOptions(),
}
for _, opt := range opts {
if opt == nil {
return nil, errors.New("chat: nil option")
}
opt(&cfg)
}
if cfg.state == nil {
return nil, errors.New("chat: runtime state is required")
}
if len(cfg.adapters) == 0 {
return nil, errors.New("chat: at least one adapter is required")
}
if cfg.logger == nil {
return nil, errors.New("chat: logger is required")
}
if cfg.observer == nil {
cfg.observer = noopObserver{}
}
if err := validateRuntimeOptions(cfg.options); err != nil {
return nil, err
}
baseCtx, baseCancel := context.WithCancel(context.Background())
chat := &Chat{
state: cfg.state,
adapters: map[string]Adapter{},
logger: cfg.logger,
observer: cfg.observer,
options: cfg.options,
eventAcceptances: map[string]*eventAcceptance{},
baseCtx: baseCtx,
baseCancel: baseCancel,
pending: map[string]*pendingWaiter{},
burstScopes: map[string]*burstScope{},
}
if cfg.options.Concurrency == ConcurrencyConcurrent {
chat.concurrencySlots = make(chan struct{}, cfg.options.MaxConcurrent)
}
if cfg.options.Dispatch == DispatchDeferred {
chat.admission = newAdmissionGate(cfg.options.MaxDetached, cfg.options.MaxDetachedPerTenant)
}
for _, adapter := range cfg.adapters {
if adapter == nil {
baseCancel()
return nil, errors.New("chat: nil adapter")
}
name := adapter.Name()
if name == "" {
baseCancel()
return nil, errors.New("chat: adapter name is required")
}
if _, exists := chat.adapters[name]; exists {
baseCancel()
return nil, fmt.Errorf("chat: adapter %q registered more than once", name)
}
chat.adapters[name] = adapter
if err := adapter.Init(ctx); err != nil {
baseCancel()
return nil, errors.Join(
fmt.Errorf("chat: initialize adapter %q: %w", name, err),
shutdownAdapters(ctx, chat.adapters),
)
}
}
return chat, nil
}
func validateRuntimeOptions(options RuntimeOptions) error {
if options.DedupeTTL <= 0 {
return errors.New("chat: dedupe ttl must be positive")
}
if options.ThreadLockTTL <= 0 {
return errors.New("chat: thread lock ttl must be positive")
}
switch options.Concurrency {
case ConcurrencyDrop, ConcurrencyQueue:
case ConcurrencyDebounce:
if options.DebounceInterval <= 0 {
return errors.New("chat: debounce interval must be positive under the debounce strategy")
}
if options.Dispatch != DispatchDeferred {
return errors.New("chat: debounce strategy requires deferred dispatch")
}
// The quiet-period wait runs inside the DetachTimeout-bounded Detached
// Work Context: a timeout at or below the interval would abandon every
// accepted event before its quiet period closes.
if options.DetachTimeout <= options.DebounceInterval {
return errors.New("chat: detach timeout must exceed the debounce interval under the debounce strategy")
}
case ConcurrencyConcurrent:
if options.MaxConcurrent <= 0 {
return errors.New("chat: max concurrent must be positive under the concurrent strategy")
}
case ConcurrencyBurst:
if options.BurstWindow <= 0 {
return errors.New("chat: burst window must be positive under the burst strategy")
}
if options.MaxBurstBatch < 0 {
return errors.New("chat: max burst batch must not be negative under the burst strategy")
}
if options.Dispatch != DispatchDeferred {
return errors.New("chat: burst strategy requires deferred dispatch")
}
default:
return errors.New("chat: unsupported concurrency strategy")
}
switch options.LockScope {
case LockScopeThread, LockScopeChannel:
default:
return errors.New("chat: unsupported lock scope")
}
switch options.Dispatch {
case DispatchSync:
case DispatchDeferred:
if options.DetachTimeout <= 0 {
return errors.New("chat: detach timeout must be positive under deferred dispatch")
}
if options.MaxDetached <= 0 {
return errors.New("chat: max detached must be positive under deferred dispatch")
}
if options.MaxDetachedPerTenant < 0 {
return errors.New("chat: max detached per tenant must not be negative under deferred dispatch")
}
default:
return errors.New("chat: unsupported dispatch mode")
}
return nil
}
// OnNewMention installs or atomically replaces the single new-mention handler.
// This intentionally differs from Vercel Chat SDK's multiple-handler hooks.
func (c *Chat) OnNewMention(handler MessageHandler) {
assert(c != nil, "OnNewMention called on nil runtime")
c.handlersMu.Lock()
defer c.handlersMu.Unlock()
c.newMention = handler
}
// OnSubscribedMessage installs or atomically replaces the single subscribed-message handler.
// This intentionally differs from Vercel Chat SDK's multiple-handler hooks.
func (c *Chat) OnSubscribedMessage(handler MessageHandler) {
assert(c != nil, "OnSubscribedMessage called on nil runtime")
c.handlersMu.Lock()
defer c.handlersMu.Unlock()
c.subscribedMessage = handler
}
// OnCommand installs or atomically replaces the single command handler. A Command
// Event routes here and never to the message hooks, even in a Subscribed Thread.
// This intentionally differs from Vercel Chat SDK's multiple-handler hooks.
func (c *Chat) OnCommand(handler CommandHandler) {
assert(c != nil, "OnCommand called on nil runtime")
c.handlersMu.Lock()
defer c.handlersMu.Unlock()
c.command = handler
}
// OnInteraction installs or atomically replaces the single interaction handler. An
// Interaction Event routes here and never to the message hooks. This intentionally
// differs from Vercel Chat SDK's multiple-handler hooks.
func (c *Chat) OnInteraction(handler InteractionHandler) {
assert(c != nil, "OnInteraction called on nil runtime")
c.handlersMu.Lock()
defer c.handlersMu.Unlock()
c.interaction = handler
}
func (c *Chat) Webhook(adapterName string) (http.Handler, error) {
assert(c != nil, "Webhook called on nil runtime")
adapter, ok := c.adapters[adapterName]
if !ok {
return nil, fmt.Errorf("chat: adapter %q is not registered", adapterName)
}
return adapter.Webhook(c.dispatch), nil
}
func (c *Chat) Thread(ctx context.Context, id ThreadID) (*Thread, error) {
assert(c != nil, "Thread called on nil runtime")
if err := ctx.Err(); err != nil {
return nil, err
}
name, err := adapterNameFromThreadID(id)
if err != nil {
return nil, err
}
adapter, ok := c.adapters[name]
if !ok {
return nil, fmt.Errorf("chat: adapter %q is not registered", name)
}
ref, err := adapter.ValidateThreadID(id)
if err != nil {
return nil, fmt.Errorf("chat: validate thread id: %w", err)
}
return c.newThread(adapter, ref), nil
}
func AdapterAs[T any](c *Chat, adapterName string) (T, bool) {
var zero T
if c == nil {
return zero, false
}
adapter, ok := c.adapters[adapterName]
if !ok {
return zero, false
}
typed, ok := adapter.(T)
return typed, ok
}
func (c *Chat) Shutdown(ctx context.Context) error {
assert(c != nil, "Shutdown called on nil runtime")
c.shutdownMu.Lock()
if c.shutdown {
done := c.shutdownDone
c.shutdownMu.Unlock()
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
c.shutdown = true
done := make(chan struct{})
c.shutdownDone = done
c.shutdownMu.Unlock()
defer close(done)
// Close admission first so a delivery racing Shutdown is rejected with
// ErrAdmissionRejected (the platform's retry covers it) instead of being
// admitted into a runtime about to cancel its work. Then cancel detached
// tails and drain (bounded by ctx) before shutting down adapters and state.
//
// The drain waits on admission slots before the tail WaitGroup: a delivery
// that won the admission race is retained from admit until its tail
// goroutine returns, so waiting for every slot covers deliveries still in
// their synchronous prelude — which hold no WaitGroup count yet — and
// guarantees no tail is spawned (and no WaitGroup Add happens) after the
// slot drain completes.
var admissionDrained <-chan struct{}
if c.admission != nil {
admissionDrained = c.admission.close()
}
c.baseCancel()
var drainErr error
drained := make(chan struct{})
go func() {
if admissionDrained != nil {
<-admissionDrained
}
c.inflight.Wait()
close(drained)
}()
select {
case <-drained:
case <-ctx.Done():
drainErr = ctx.Err()
}
err := errors.Join(drainErr, shutdownAdapters(ctx, c.adapters))
if stateErr := c.state.Shutdown(ctx); stateErr != nil {
return errors.Join(err, fmt.Errorf("shutdown state: %w", stateErr))
}
return err
}
func shutdownAdapters(ctx context.Context, adapters map[string]Adapter) error {
var errs []error
for name, adapter := range adapters {
if err := adapter.Shutdown(ctx); err != nil {
errs = append(errs, fmt.Errorf("shutdown adapter %q: %w", name, err))
}
}
return errors.Join(errs...)
}
func (c *Chat) dispatch(ctx context.Context, event *Event) error {
// The admission sequence is assigned before any prelude work so pending
// registration can order deliveries by admission: a delivery delayed in
// validation/dedupe/routing can never displace a newer waiter.
seq := c.dispatchSeq.Add(1)
if c.options.Dispatch == DispatchDeferred {
return c.dispatchDeferred(ctx, event, seq)
}
return c.dispatchSync(ctx, event, seq)
}
// dispatchSync runs the prelude and the routed handler inline under the request
// context, releasing the Thread Lock when the handler returns. Under the queue
// strategy a Lock Conflict waits inline (bounded by ctx) for the in-flight
// handler before the routed handler runs. Under the concurrent strategy the
// request waits inline for a MaxConcurrent slot instead of a lock.
func (c *Chat) dispatchSync(ctx context.Context, event *Event, seq uint64) error {
work, resolved, err := c.prelude(ctx, event, seq)
if err != nil || resolved {
return err
}
if work.noLock {
if !c.acquireConcurrencySlot(ctx, event) {
c.safeEnd(work.span, OutcomeIgnored, RouteAttr(work.route))
return nil
}
defer c.releaseConcurrencySlot()
} else {
if work.needsLock {
// The steerability hook requires deferred dispatch, so a sync wait
// never carries an ownership reservation.
lease, outcome := c.queueForLock(ctx, work.scope, event, work.waitLabel)
if outcome != acquireHeld {
c.safeEnd(work.span, waitOutcome(outcome), RouteAttr(work.route))
return nil
}
work.lease = lease
}
defer c.releaseLock(ctx, work.lease, event.ThreadID)
}
if err := work.run(ctx); err != nil {
c.logger.Error("chat handler failed", "error", err, "adapter", event.Adapter, "event_id", event.ID, "route", work.route)
c.safeEvent(ctx, ObsHandlerError, AdapterAttr(event.Adapter), RouteAttr(work.route))
c.safeEnd(work.span, OutcomeError, RouteAttr(work.route))
return nil
}
c.safeEnd(work.span, OutcomeHandled, RouteAttr(work.route))
return nil
}
// dispatchDeferred runs the prelude under the request context and, on a routed
// event, launches the detached tail and returns so the adapter can acknowledge
// the platform (ack-then-work). The Admission Bound gate runs first, before
// any acknowledgement and before dedupe marking: a delivery rejected at the
// cap fails fast with ErrAdmissionRejected, is never marked in Event Identity
// (so a platform retry is not deduped away), and does no State work at all —
// under saturation even a redelivered duplicate receives the overload
// response, converging to the ordinary duplicate acknowledgement once
// capacity frees (ADR 0015).
func (c *Chat) dispatchDeferred(ctx context.Context, event *Event, seq uint64) error {
if err := validateEvent(event); err != nil {
return err
}
assert(c.admission != nil, "deferred dispatch requires the admission gate")
release, admitted := c.admission.admit(event.Adapter, event.Tenant)
if !admitted {
spanCtx, span := c.safeDispatch(ctx, AdapterAttr(event.Adapter), TenantAttr(event.Tenant))
c.logger.Warn("chat deferred dispatch admission rejected", "adapter", event.Adapter, "tenant", event.Tenant, "event_id", event.ID)
c.safeEvent(spanCtx, ObsAdmissionRejected, AdapterAttr(event.Adapter), TenantAttr(event.Tenant))
c.safeEnd(span, OutcomeAdmissionRejected)
return fmt.Errorf("chat: deferred dispatch admission rejected for adapter %q: %w", event.Adapter, ErrAdmissionRejected)
}
work, resolved, err := c.prelude(ctx, event, seq)
if err != nil || resolved {
// The delivery retains nothing past the prelude: an errored or
// resolved (duplicate, dropped, ignored, unrouted) prelude frees its
// admission slot immediately.
release()
return err
}
work.releaseAdmission = release
if work.burst {
// A burst member parks in its scope's collection window instead of
// owning a detached tail; its admission slot travels with it and the
// scope's runner releases it at the member's terminal disposition, so
// a parked member counts against MaxDetached for as long as its
// payload and closure are retained.
c.joinBurstBatch(work)
return nil
}
c.startDetachedTail(work)
return nil
}
// preludeWork carries the routing decision from the prelude to the dispatch tail
// for a routed event.
type preludeWork struct {
event *Event
lease LockLease
// run invokes the routed handler with its typed input (MessageEvent,
// CommandEvent, or InteractionEvent); keeping it handler-type agnostic lets the
// dispatch tails stay identical across Event kinds.
run func(context.Context) error
route string
scope string
// needsLock is true when the prelude did not acquire the Thread Lock (a
// queued Lock Conflict, or the debounce strategy); the tail must wait for
// and acquire it before running the handler.
needsLock bool
// waitLabel names the coordination mode ("queue" or "debounce") in wait
// observations so a log line names the strategy that produced it.
waitLabel string
// displaced closes when a newer event supersedes this pending waiter; the
// debounce quiet-period wait exits promptly on it instead of parking through
// its full interval.
displaced <-chan struct{}
// debounce is true when the tail must hold the event through the
// DebounceInterval quiet period before waiting for the lock.
debounce bool
// noLock is true under the concurrent strategy: no Thread Lock is taken and
// the run is bounded by a MaxConcurrent slot instead.
noLock bool
// burst is true under the burst strategy: the routed event joins its
// scope's collection window as a batch member instead of owning a detached
// tail; the scope's runner owns its span, its admission slot, and its
// terminal disposition.
burst bool
// releaseAdmission frees the delivery's Admission Bound slot. It is set
// only under DispatchDeferred and runs when the detached tail goroutine
// returns — not when the handler returns — so stalled cleanup (lock
// release, lease refresh drain) still counts as retention.
releaseAdmission func()
// span is opened in the prelude and closed by the tail, so deferred dispatch
// measures Ack-Then-Work latency to handler completion.
span DispatchSpan
}
// prelude runs the synchronous portion of dispatch before ack: open the dispatch
// span, validate, dedupe, acquire the Thread Lock, validate the thread id, filter
// nil/self events, and route. A resolved event (duplicate, dropped Lock Conflict,
// ignored, unrouted) returns resolved=true with no work and closes the span with
// its terminal outcome. A failed prelude returns the error and, as today, leaves
// the event un-marked so a retry is not deduped away; its span is closed with the
// error outcome.
//
// Routing precedence: command-ness and interaction-ness are Event kinds and take
// precedence over subscription state, so a command/interaction in a Subscribed
// Thread still routes to its own hook, never to the message hooks. Only an Event
// with no Message, Command, or Interaction payload remains an Ignored Event.
//
// Under the queue strategy a Lock Conflict on a routed event does not block: the
// event is registered as pending (no lease) and returned with needsLock=true so
// the tail acquires the lock, keeping ack prompt under DispatchDeferred.
func (c *Chat) prelude(ctx context.Context, event *Event, seq uint64) (preludeWork, bool, error) {
if err := validateEvent(event); err != nil {
return preludeWork{}, true, err
}
spanCtx, span := c.safeDispatch(ctx, AdapterAttr(event.Adapter), TenantAttr(event.Tenant))
ctx = spanCtx
acceptance, primary := c.beginEventAcceptance(event.ID)
if !primary {
// The primary owns the terminal observation for the shared Event Identity.
err := waitEventAcceptance(ctx, acceptance)
c.safeEnd(span, OutcomeDuplicate)
return preludeWork{}, true, err
}
finish := func(err error) error {
c.finishEventAcceptance(event.ID, acceptance, err)
return err
}
acceptEvent := func() (bool, error) {
firstSeen, err := c.markAcceptedEvent(ctx, event)
if err != nil {
return false, finish(err)
}
c.finishEventAcceptance(event.ID, acceptance, nil)
if !firstSeen {
c.safeEvent(ctx, ObsDedupeHit, AdapterAttr(event.Adapter))
}
return firstSeen, nil
}
adapter, ok := c.adapters[event.Adapter]
if !ok {
err := finish(fmt.Errorf("chat: event adapter %q is not registered", event.Adapter))
c.safeEnd(span, OutcomeError)
return preludeWork{}, true, err
}
// The Thread ID is validated before the Thread Lock so the lock scope key
// (which may be channel-wide) comes from the adapter-validated ThreadRef; an
// event with an invalid Thread ID never touches the lock.
ref, validateErr := adapter.ValidateThreadID(event.ThreadID)
if validateErr != nil {
err := finish(fmt.Errorf("chat: validate event thread id: %w", validateErr))
c.safeEnd(span, OutcomeError)
return preludeWork{}, true, err
}
scope := c.lockScopeKey(event, ref)
// The prelude acquires the Thread Lock only under drop/queue; debounce and
// burst always coordinate in the tail (keeping ack prompt and free of lock
// contention), and concurrent takes no lock at all.
var lease LockLease
conflicted := false
switch c.options.Concurrency {
case ConcurrencyDrop, ConcurrencyQueue:
acquiredLease, acquired, err := c.state.AcquireLock(ctx, scope, c.options.ThreadLockTTL)
if err != nil {
err := finish(fmt.Errorf("chat: acquire thread lock: %w", err))
c.safeEnd(span, OutcomeError)
return preludeWork{}, true, err
}
lease = acquiredLease
if !acquired {
if c.options.Concurrency == ConcurrencyDrop {
accepted, err := acceptEvent()
if err != nil {
c.safeEnd(span, OutcomeError)
return preludeWork{}, true, err
}
if !accepted {
c.safeEnd(span, OutcomeDuplicate)
return preludeWork{}, true, nil
}
c.logger.Info("chat lock conflict dropped", "adapter", event.Adapter, "event_id", event.ID, "thread_id", event.ThreadID)
c.safeEvent(ctx, ObsLockConflict, AdapterAttr(event.Adapter))
c.safeEnd(span, OutcomeDroppedLockConflict)
return preludeWork{}, true, nil
}
conflicted = true
}
case ConcurrencyDebounce, ConcurrencyBurst, ConcurrencyConcurrent:
}
// releaseOnResolve releases the lease when the event resolves here; an event
// whose strategy defers lock coordination to the tail holds no lease yet.
releaseOnResolve := func() {
if lease == (LockLease{}) {
return
}
c.releaseLock(ctx, lease, event.ThreadID)
}
// resolveError releases the lease and closes the span for a failed prelude.
resolveError := func(err error) (preludeWork, bool, error) {
releaseOnResolve()
c.safeEnd(span, OutcomeError)
return preludeWork{}, true, finish(err)
}
// resolveIgnored accepts (deduping), logs, and closes the span for an Ignored
// Event; a duplicate closes as OutcomeDuplicate instead.
resolveIgnored := func(reason, logMsg string, level slog.Level) (preludeWork, bool, error) {
releaseOnResolve()
accepted, err := acceptEvent()
if err != nil {
c.safeEnd(span, OutcomeError)
return preludeWork{}, true, err
}
if !accepted {
c.safeEnd(span, OutcomeDuplicate)
return preludeWork{}, true, nil
}
c.logger.Log(ctx, level, logMsg, "adapter", event.Adapter, "event_id", event.ID)
c.safeEvent(ctx, ObsIgnoredEvent, AdapterAttr(event.Adapter), ReasonAttr(reason))
c.safeEnd(span, OutcomeIgnored, ReasonAttr(reason))
return preludeWork{}, true, nil
}
thread := c.newThread(adapter, ref)
bot := adapter.BotActor()
// Non-message routing precedence: a Command Event or Interaction Event routes
// to its own single-slot hook regardless of subscription state. Self-issued
// commands/interactions are filtered like self messages so a bot cannot loop.
switch {
case event.Command != nil:
if isSelfActor(event.Command.Actor, bot) {
return resolveIgnored("self-command", "chat ignored self command", slog.LevelDebug)
}
c.handlersMu.RLock()
handler := c.command
c.handlersMu.RUnlock()
if handler == nil {
return resolveIgnored("no-command-handler", "chat ignored command with no handler", slog.LevelInfo)
}
cmdEvent := &CommandEvent{Event: event, Thread: thread, Command: event.Command}
return c.routedWork(ctx, span, event, lease, scope, seq, conflicted, "command", func(ctx context.Context) error {
return handler(ctx, cmdEvent)
}, acceptEvent, releaseOnResolve)
case event.Interaction != nil:
if isSelfActor(event.Interaction.Actor, bot) {
return resolveIgnored("self-interaction", "chat ignored self interaction", slog.LevelDebug)
}
c.handlersMu.RLock()
handler := c.interaction
c.handlersMu.RUnlock()
if handler == nil {
return resolveIgnored("no-interaction-handler", "chat ignored interaction with no handler", slog.LevelInfo)
}
intEvent := &InteractionEvent{Event: event, Thread: thread, Interaction: event.Interaction}
return c.routedWork(ctx, span, event, lease, scope, seq, conflicted, "interaction", func(ctx context.Context) error {
return handler(ctx, intEvent)
}, acceptEvent, releaseOnResolve)
}
if event.Message == nil {
return resolveIgnored("non-message", "chat ignored non-message event", slog.LevelInfo)
}
if isSelfActor(event.Message.Author, bot) {
return resolveIgnored("self-message", "chat ignored self message", slog.LevelDebug)
}
handler, route, err := c.route(ctx, event)
if err != nil {
return resolveError(err)
}
if handler == nil {
return resolveIgnored("unrouted", "chat ignored unrouted message", slog.LevelInfo)
}
msgEvent := &MessageEvent{Event: event, Thread: thread, Message: event.Message}
return c.routedWork(ctx, span, event, lease, scope, seq, conflicted, route, func(ctx context.Context) error {
return handler(ctx, msgEvent)
}, acceptEvent, releaseOnResolve)
}
// routedWork accepts a routed event (deduping), applies the Concurrency
// Strategy's coordination decision (queue/debounce/concurrent), and builds the
// handler-agnostic preludeWork. A duplicate or a dedupe error resolves here
// instead.
func (c *Chat) routedWork(
ctx context.Context,
span DispatchSpan,
event *Event,
lease LockLease,
scope string,
seq uint64,
conflicted bool,
route string,
run func(context.Context) error,
acceptEvent func() (bool, error),
releaseOnResolve func(),
) (preludeWork, bool, error) {
accepted, err := acceptEvent()
if err != nil {
releaseOnResolve()
c.safeEnd(span, OutcomeError)
return preludeWork{}, true, err
}
if !accepted {
releaseOnResolve()
c.safeEnd(span, OutcomeDuplicate)
return preludeWork{}, true, nil
}
work := preludeWork{
event: event,
lease: lease,
run: run,
route: route,
scope: scope,
span: span,
}
// resolveStale closes a routed event whose delivery was admitted before the
// scope's current pending waiter: it never displaces the newer waiter.
resolveStale := func(label string) (preludeWork, bool, error) {
c.logger.Debug("chat "+label+" waiter superseded", "adapter", event.Adapter, "event_id", event.ID, "thread_id", event.ThreadID)
c.safeEnd(span, OutcomeIgnored, RouteAttr(route))
return preludeWork{}, true, nil
}
switch c.options.Concurrency {
case ConcurrencyDrop, ConcurrencyQueue:
if conflicted {
// Only queue reaches routing with a conflict (drop resolves at the
// conflict site): the event waits as its scope's pending waiter.
work.needsLock = true
work.waitLabel = "queue"
displaced, registered := c.registerPending(scope, event, seq, work.waitLabel)
if !registered {
return resolveStale(work.waitLabel)
}
work.displaced = displaced
}
return work, false, nil
case ConcurrencyDebounce:
work.needsLock = true
work.debounce = true
work.waitLabel = "debounce"
displaced, registered := c.registerPending(scope, event, seq, work.waitLabel)
if !registered {
return resolveStale(work.waitLabel)
}
work.displaced = displaced
return work, false, nil
case ConcurrencyConcurrent:
work.noLock = true
return work, false, nil
case ConcurrencyBurst:
// The routed event becomes a burst member: dispatchDeferred parks it
// in its scope's collection window with its admission slot attached.
// Burst requires deferred dispatch (validated at construction), so
// dispatchSync never sees this flag.
assert(c.options.Dispatch == DispatchDeferred, "burst strategy requires deferred dispatch")
work.burst = true
return work, false, nil
}
// Unreachable: the strategy set is validated at construction.
releaseOnResolve()
c.safeEnd(span, OutcomeError)
return preludeWork{}, true, fmt.Errorf("chat: unsupported concurrency strategy %d", c.options.Concurrency)
}
// startDetachedTail runs the routed handler on the detached work context after
// ack: the Thread Lock is held across the tail, refreshed via ExtendLock, and
// released on exit. The context is derived from baseCtx, bounded by
// DetachTimeout, and cancelled by Shutdown. When needsLock, the tail first waits
// for the lock (after the debounce quiet period when the work asks for one); a
// superseded or abandoned waiter exits without running the handler. Concurrent
// tails wait for a MaxConcurrent slot rather than a lock.
func (c *Chat) startDetachedTail(work preludeWork) {
assert(work.releaseAdmission != nil, "detached tail requires a held admission slot")
tailCtx, tailCancel := context.WithTimeout(c.baseCtx, c.options.DetachTimeout)
c.inflight.Add(1)
go func() {
// The admission slot is held until this goroutine returns: everything
// the tail retains — the parked wait, the handler run, and cleanup
// (lock release, refresh-loop drain) — counts against MaxDetached.
defer work.releaseAdmission()
defer c.inflight.Done()
defer tailCancel()
if work.noLock {
if !c.acquireConcurrencySlot(tailCtx, work.event) {
c.safeEnd(work.span, OutcomeIgnored, RouteAttr(work.route))
return
}
defer c.releaseConcurrencySlot()
c.logger.Info("chat deferred dispatch started", "adapter", work.event.Adapter, "event_id", work.event.ID, "route", work.route)
err := work.run(tailCtx)
c.endHandlerRun(tailCtx, work.event, work.route, work.span, err)
return
}
if work.debounce && !c.waitDebounceQuietPeriod(tailCtx, work) {
return
}